sps_bill 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3@sps_bill_scanner --create

data/.travis.yml

@@ -0,0 +1,3 @@
+# These are specific configuration settings required for travis-ci
+# see http://travis-ci.org/tardate/sps_bill_scanner
+rvm: 1.9.3

data/Gemfile

@@ -0,0 +1,18 @@
+source "http://rubygems.org"
+
+gem 'pdf-reader', '1.1.1'
+gem 'getoptions', '~> 0.3'
+
+group :development do
+  gem 'bundler', '~> 1.1.4'
+  gem 'jeweler', '~> 1.6.4'
+  gem 'rcov', '>= 0'
+end
+
+group :development, :test do
+  gem 'rake', '~> 0.9.2.2'
+  gem 'rspec', '~> 2.8.0', :require => 'spec'
+  gem 'rdoc', '~> 3.11'
+  # guard for auto-running tests
+  gem 'guard-rspec'
+end

data/Gemfile.lock

@@ -0,0 +1,58 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    Ascii85 (1.0.1)
+    diff-lcs (1.1.3)
+    ffi (1.0.11)
+    getoptions (0.3)
+    git (1.2.5)
+    guard (1.2.3)
+      listen (>= 0.4.2)
+      thor (>= 0.14.6)
+    guard-rspec (1.2.0)
+      guard (>= 1.1)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    json (1.6.4)
+    listen (0.4.7)
+      rb-fchange (~> 0.0.5)
+      rb-fsevent (~> 0.9.1)
+      rb-inotify (~> 0.8.8)
+    pdf-reader (1.1.1)
+      Ascii85 (~> 1.0.0)
+      ruby-rc4
+    rake (0.9.2.2)
+    rb-fchange (0.0.5)
+      ffi
+    rb-fsevent (0.9.1)
+    rb-inotify (0.8.8)
+      ffi (>= 0.5.0)
+    rcov (0.9.11)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    ruby-rc4 (0.1.5)
+    thor (0.15.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.1.4)
+  getoptions (~> 0.3)
+  guard-rspec
+  jeweler (~> 1.6.4)
+  pdf-reader (= 1.1.1)
+  rake (~> 0.9.2.2)
+  rcov
+  rdoc (~> 3.11)
+  rspec (~> 2.8.0)

data/Guardfile

@@ -0,0 +1,10 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec', :version => 2, :all_on_start => true, :all_after_pass => false do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  ## we're not watching source files
+end
+

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Paul Gallagher
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,180 @@
+= SP Services Bill Scanner {<img src="https://secure.travis-ci.org/tardate/sps_bill_scanner.png" />}[http://travis-ci.org/tardate/sps_bill_scanner]
+
+Extracts bill details from SP Services PDF bills so that you can, um, do geeky data analysis n'stuff.
+
+If you are an SP Services subscriber, download your bills from https://services.spservices.sg
+
+If you are not an SP Services subscriber, this gem ain't going to be much use for you!
+
+Some example analysis using {R}[http://www.r-project.org/] is included in the <tt>scripts</tt> folder.
+
+== Requirements and Known Limitations
+
+* Requires Ruby 1.9 (1.8 compatibility not tested or assured)
+* R is used for data analysis examples - this is optional
+* Currently it does not handle multi-property bills.
+* May not handle extensive multi-page bills for a single property correctly.
+
+If you _do_ come across bills that this gem can't read correctly, your help to get it fixed is greatly needed:
+either submit a fix yourself, or report the problem at https://github.com/tardate/sps_bill_scanner/issues
+
+== Help Required: Test this with your own bills
+
+Unfortunately, there isn't a definitive set of bill examples that make it possible to ensure that
+this gem will work for every bill issued by SP Services. And since bills are in PDF format,
+extracting the data in a structured manner is a task fraught with pitfalls.
+
+I have tested this with my own bills (going back over a year) but that definitely doesn't mean it will
+work for others.
+
+So I need help from others who are willing to test with their own bills. I'm not asking for your bills
+as that raises privacy concerns, and I definitely don't want real bills committed to the git repository.
+
+Instead, I have setup the tests in a way that should make it easy for you to test with your own bills.
+
+Here's the basic outline:
+
+* First, make sure tests are running green for you 'as-is':
+
+  - fork or clone the repo
+  - <tt>bundle</tt> will install development dependencies
+  - <tt>rake</tt> will run the tests - they should all be OK
+
+* Get you own PDF bills from from https://services.spservices.sg
+
+  - put them in <tt>spec/fixtures/personal_pdf_samples</tt>
+  - NB: these are ignored by git so you won't accidentally commit them
+
+* At this point you can run the tests with <tt>rake</tt> and it will do a very basic
+  check of the PDFs you have added
+* To run complete checks to ensure all the data is being extracted correctly:
+
+  - see the doc in <tt>spec/fixtures/personal_pdf_samples/expectations.yml.sample</tt>
+  - copy this to <tt>spec/fixtures/personal_pdf_samples/expectations.yml</tt>
+  - enter in the details that describe each bill you have added
+  - now when you run <tt>rake</tt> it will also verify the data extracted from your
+    bills using expectations.yml
+
+Feel free to get in touch or discuss in the github issues area if you are trying to help but run
+into problems with this!
+
+== Installation
+
+  gem install sps_bill
+
+== Command Line Usage
+
+Once the gem is installed, use <tt>sps_bill</tt> at the command line to interact with the library manually.
+
+To get help on command options:
+
+  $ sps_bill -h
+
+For example: to extract all data in CSV format from a set of PDF bills:
+
+  $ sps_bill --data=all ./path_to/my_bills*.pdf
+
+== Programmatic Usage
+
+You can use the gem from your own scripts or applications. There are just two classes you really need
+to understand:
+
+* SpsBill::BillCollection is an Array-like class that represents a collection of bills.
+  - the <tt>load</tt> method is used to initialise it given a path or array of filenames
+  - a range of collection methods are provided to extract sets of data (e.g. <tt>electricity_usages</tt>)
+* SpsBill::Bill represents an individual bill
+  - initialised given a file name
+  - provides a range of accessors to get at individual data elements (e.g. <tt>electricity_usage</tt>)
+
+To load a collection of bills:
+
+  > require 'sps_bill'
+  > bills = SpsBill::BillCollection.load('./my_bills/*.pdf')
+  > bills.total_amounts
+    => [["2011-10-01", 168.86], ["2011-11-01", 196.46], ["2011-12-01", 176.54]]
+  > bills.electricity_usages
+    => [["2011-10-01", 14.0, 0.2728, 3.82], ["2011-10-01", 444.0, 0.2698, 119.79],
+    ["2011-11-01", 2.0, 0.2728, 0.54], ["2011-11-01", 537.0, 0.2698, 144.88],
+    ["2011-12-01", 482.0, 0.2698, 130.04]]
+  > bills.gas_usages
+    => [["2011-10-01", 12.0, 0.1961, 2.35], ["2011-11-01", 12.0, 0.2117, 2.54], ["2011-12-01", 12.0, 0.2117, 2.54]]
+  > bills.water_usages
+    => [["2011-10-01", 8.4, 1.17, 9.83], ["2011-11-01", 11.4, 1.17, 13.34], ["2011-12-01", 9.6, 1.17, 11.23]]
+
+
+To load and examine a specific bill:
+
+  > require 'sps_bill'
+  > pdf_bill_file = "./my_latest_bill.pdf"
+  > bill = SpsBill::Bill.new(pdf_bill_file)
+  > bill.account_number
+  8123123123
+  > bill.total_amount
+  251.44
+  > bill.invoice_date
+  2011-05-31
+  > bill.invoice_month
+  2011-05-01
+  > bill.electricity_usage
+  [{:kwh=>4.0, :rate=>0.241, :amount=>0.97},{:kwh=>616.0, :rate=>0.2558, :amount=>157.57}]
+  > bill.gas_usage
+  [{:kwh=>18.0, :rate=>0.1799, :amount=>3.24}]
+  > bill.water_usage
+  [{:cubic_m=>36.1, :rate=>1.17, :amount=>42.24},{:cubic_m=>-3.0, :rate=>1.4, :amount=>-4.2}]
+  > bill.to_s
+  Account number: 8123123123
+  Invoice date  : 2011-10-31
+  Service month : 2011-10-01
+  Total bill    : $168.86
+
+  Electricity Usage
+  -----------------
+  [{:kwh=>14.0, :rate=>0.2728, :amount=>3.82}, {:kwh=>444.0, :rate=>0.2698, :amount=>119.79}]
+
+  Gas Usage
+  ---------
+  [{:kwh=>12.0, :rate=>0.1961, :amount=>2.35}]
+
+  Water Usage
+  -----------
+  [{:cubic_m=>8.4, :rate=>9.83, :amount=>0.0}, {:cubic_m=>1.17, :rate=>0.0, :amount=>0.0}]
+
+== Data Analysis with R
+
+Some examples of bill data and analysis using {R}[http://www.r-project.org/] are included
+in the <tt>scripts</tt> folder.
+
+=== sample scripts
+
+[scripts/scan_all_bills.sh]  an example script to scan a set of bills and produce a csv file for analysis
+[scripts/full_analysis.R]  an example script that prepares a one-page PDF summary analysis
+
+=== sample data and analysis
+
+[data/all_services.csv.sample]  sample CSV data for a years worth of elec, gas, and water
+[data/all_services.sample.pdf]  PDF analysis produced by this script for all_services.csv.sample
+[data/elec_and_water_only.csv.sample]  sample CSV data for a years worth of elec and water
+[data/elec_and_water_only.sample.pdf]  PDF analysis produced by this script for elec_and_water_only.csv.sample
+
+=== example run
+
+  ./scan_all_bills.sh ../path_to_my_bills/*.pdf > my_bill_data.csv
+  ./full_analysis.R data_file.csv my_bill_data.csv
+
+This will have produced an analysis of all your bills in <tt>full_analysis.pdf</tt>.
+
+== Contributing to sps_bill_scanner
+
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2012 Paul Gallagher. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,49 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+require 'rspec'
+require 'rspec/core/rake_task'
+
+$LOAD_PATH.unshift('lib')
+require 'sps_bill/version'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "sps_bill"
+  gem.version = SpsBill::Version::STRING
+  gem.homepage = "https://github.com/tardate/sps_bill_scanner"
+  gem.license = "MIT"
+  gem.summary = %Q{SP Services PDF bill structured data reader}
+  gem.description = %Q{a library that can read SP Services PDF bills and extract and summarize the bill details}
+  gem.email = "gallagher.paul@gmail.com"
+  gem.authors = ["Paul Gallagher"]
+  gem.files.exclude 'pkg/*'
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+desc "Run all RSpec test examples"
+RSpec::Core::RakeTask.new do |spec|
+  spec.rspec_opts = ["-c", "-f progress"]
+  spec.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => :spec
+
+require 'rdoc/task'
+RDoc::Task.new do |rdoc|
+  rdoc.main = "README.rdoc"
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sps_bill #{SpsBill::Version::STRING}"
+  rdoc.rdoc_files.include('README*', 'lib/**/*.rb')
+end

data/bin/sps_bill

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'sps_bill'
+require 'getoptions'
+
+begin
+  SpsBill::Shell.new(GetOptions.new(SpsBill::Shell::OPTIONS)).run
+rescue Exception => e
+  STDERR.puts e
+end

data/lib/pdf/object_hash.rb

@@ -0,0 +1,39 @@
+class PDF::Reader
+  class ObjectHash
+
+    def extract_io_from(input)
+      if input.respond_to?(:seek) && input.respond_to?(:read)
+        input
+      elsif File.file?(input.to_s)
+        read_with_quirks(input)
+      else
+        raise ArgumentError, "input must be an IO-like object or a filename"
+      end
+    end
+
+    # Load file as a StringIO stream, accounting for invalid format
+    # where additional characters exist in the file before the %PDF start of file
+    def read_with_quirks(input)
+      stream = File.open(input.to_s, "rb")
+      if ofs = pdf_offset(stream)
+        stream.seek(ofs)
+        StringIO.new(stream.read)
+      else
+        raise ArgumentError, "invalid file format"
+      end
+    end
+    private :read_with_quirks
+
+    # Returns the offset of the PDF document in the +stream+.
+    # Checks up to 50 chars into the file, returns nil of no PDF stream detected.
+    def pdf_offset(stream)
+      stream.rewind
+      ofs = stream.pos
+      until (c = stream.readchar) == '%' || c == 37 || ofs > 50
+        ofs += 1
+      end
+      ofs < 50 ? ofs : nil
+    end
+    private :pdf_offset
+  end
+end

data/lib/pdf/positional_text_receiver.rb

@@ -0,0 +1,16 @@
+class PDF::Reader::PositionalTextReceiver < PDF::Reader::PageTextReceiver
+
+  # record text that is drawn on the page
+  def show_text(string) # Tj
+    raise PDF::Reader::MalformedPDFError, "current font is invalid" if @state.current_font.nil?
+    newx, newy = @state.trm_transform(0,0)
+    @content[newy] ||= {}
+    @content[newy][newx] = @state.current_font.to_utf8(string)
+  end
+
+  # override content accessor
+  def content
+    @content
+  end
+
+end

data/lib/pdf/structured_reader.rb

@@ -0,0 +1,108 @@
+# Class for reading structured text content
+# This is the one that is a bit hairy - specifically check the fuzzed_y usage
+# which attempts to align text content in the PDF so it can be extracted
+# with correct alignment.
+#
+class PDF::StructuredReader
+  attr_reader :reader
+
+  # +source+ is a file name or stream-like object
+  def initialize(source)
+    @reader = PDF::Reader.new(source)
+  end
+
+  # Returns positional (with fuzzed y positioning) text content collection as a hash:
+  # { y_position: { x_position: content}}
+  def content(page=1)
+    @content ||= []
+    if @content[page]
+      @content[page]
+    else
+      @content[page] = fuzzed_y(precise_content(page))
+    end
+  end
+
+  # Returns a hash with fuzzed y positioning:
+  # { fuzzed_y_position: { x_position: content}}
+  # Given +input+ as a hash:
+  # { y_position: { x_position: content}}
+  # y values that fall within +precision+ points of another will be clustered
+  def fuzzed_y(input,precision=3)
+    output = {}
+    input.keys.sort.each do |precise_y|
+      # matching_y = (precise_y / 5.0).truncate * 5.0
+      matching_y = output.keys.select{|new_y| (new_y - precise_y).abs < precision }.first || precise_y
+      output[matching_y] ||= {}
+      output[matching_y].merge!(input[precise_y])
+    end
+    output
+  end
+
+  # Returns positional text content collection as a hash with precise x,y positioning:
+  # { y_position: { x_position: content}}
+  def precise_content(page=1)
+    @precise_content ||= []
+    if @precise_content[page]
+      @precise_content[page]
+    else
+      @precise_content[page] = load_content(page)
+    end
+  end
+
+  # Returns an array of text elements in the bounding box
+  def text_in_rect(xmin,xmax,ymin,ymax,page=1)
+    text_map = content(page)
+    box = []
+    text_map.keys.sort.reverse.each do |y|
+      if y >= ymin && y<= ymax
+        row = []
+        text_map[y].keys.sort.each do |x|
+          if x >= xmin && x<= xmax
+            row << text_map[y][x]
+          end
+        end
+        box << row unless row.empty?
+      end
+    end
+    box
+  end
+
+  # Returns the position {x: val, y: val } of +text+ on +page+
+  # +text+ may be  astring (exact match required) or a Regexp
+  def text_position(text,page=1)
+    item = if text.class <= Regexp
+      content(page).map {|k,v| if x = v.reduce(nil){|memo,vv|  memo = (vv[1] =~ text) ? vv[0] : memo  } ; [k,x] ; end }
+    else
+      content(page).map {|k,v| if x = v.rassoc(text) ; [k,x] ; end }
+    end
+    item = item.compact.flatten
+    unless item.empty?
+      { :x => item[1], :y => item[0] }
+    end
+  end
+
+  # WIP - not using Textangle yet for text extraction.
+  # Ideal usage is something like this:
+  #
+  # textangle = reader.bounding_box do
+  #   page 1
+  #   below "Electricity Services"
+  #   above "Gas Services by City Gas Pte Ltd"
+  #   right_of 240.0
+  #   left_of "Total ($)"
+  # end
+  # textangle.text
+  #
+  def bounding_box(&block)
+    PDF::Reader::Textangle.new(self,&block)
+  end
+
+  private
+
+    def load_content(page)
+      receiver = PDF::Reader::PositionalTextReceiver.new
+      reader.page(page).walk(receiver)
+      receiver.content
+    end
+
+end

data/lib/pdf/textangle.rb

@@ -0,0 +1,27 @@
+# A DSL syntax for text extraction.
+# WIP - not using this yet
+#
+# textangle = PDF::Reader::Textangle.new(reader) do
+#   page 1
+#   below "Electricity Services"
+#   above "Gas Services by City Gas Pte Ltd"
+#   right_of 240.0
+#   left_of "Total ($)"
+# end
+# textangle.text
+#
+class PDF::Reader::Textangle
+  attr_reader :reader
+  attr_writer :page,:above,:below,:left_of,:right_of
+
+  # +structured_reader+ is a PDF::StructuredReader
+  def initialize(structured_reader,&block)
+    @reader = structured_reader
+    instance_eval( &block ) if block
+  end
+
+  def text
+    # TODO
+  end
+
+end

data/lib/sps_bill/bill.rb

@@ -0,0 +1,58 @@
+# SpsBill::Bill represents an individual SP Services PDF bill
+#
+# It is initialised given a file name, and provides a range of accessors
+# to get at individual data elements (e.g. <tt>electricity_usage</tt>)
+#
+class SpsBill::Bill
+  include SpsBill::BillParser
+
+  attr_reader :source_file
+
+  # accessors for the various bill components
+  #
+  # electricity_usage charges is an array of hashed values:
+  # [{ kwh: float, rate: float, amount: float }]
+  # gas_usage charges is an array of hashed values:
+  # [{ kwh: float, rate: float, amount: float }]
+  # water_usage charges is an array of hashed values:
+  # [{ cubic_m: float, rate: float, amount: float }]
+  #
+  attr_reader :account_number,:total_amount,:invoice_date,:invoice_month
+  attr_reader :electricity_usage,:gas_usage,:water_usage
+
+  # +source+ is a file name or stream-like object
+  def initialize(source)
+    @source_file = source
+    do_complete_parse
+  end
+
+  # Returns the PDF reader isntance
+  def reader
+    @reader ||= PDF::StructuredReader.new(source_file) if source_file
+  end
+
+  # Return a pretty(-ish) text format of the core bill details
+  def to_s
+    %(
+Account number: #{account_number}
+Invoice date  : #{invoice_date}
+Service month : #{invoice_month}
+Total bill    : $#{total_amount}
+
+Electricity Usage
+-----------------
+#{electricity_usage}
+
+Gas Usage
+---------
+#{gas_usage}
+
+Water Usage
+-----------
+#{water_usage}
+
+      )
+  end
+
+
+end