sps_bill 0.1.0 → 0.1.1

data/lib/sps_bill/version.rb

@@ -2,7 +2,7 @@ module SpsBill
   class Version
     MAJOR = 0
     MINOR = 1
-    PATCH = 0
+    PATCH = 1
 
     STRING = [MAJOR, MINOR, PATCH].compact.join('.')
   end

data/scripts/data/all_services.csv.sample

@@ -0,0 +1,70 @@
+invoice_month,measure,kwh,cubic_m,rate,amount
+2011-05-01,total_charges,,,,251.44
+2011-06-01,total_charges,,,,235.7
+2011-07-01,total_charges,,,,220.05
+2011-08-01,total_charges,,,,259.2
+2011-09-01,total_charges,,,,119.97
+2011-10-01,total_charges,,,,168.86
+2011-11-01,total_charges,,,,196.46
+2011-12-01,total_charges,,,,176.54
+2012-01-01,total_charges,,,,165.74
+2012-02-01,total_charges,,,,162.64
+2012-03-01,total_charges,,,,193.2
+2012-04-01,total_charges,,,,192.48
+2012-05-01,total_charges,,,,157.31
+2012-06-01,total_charges,,,,177.63
+2011-05-01,electricity,4.0,,0.241,0.97
+2011-05-01,electricity,616.0,,0.2558,157.57
+2011-06-01,electricity,519.0,,0.2558,132.76
+2011-07-01,electricity,-14.0,,0.2558,-3.58
+2011-07-01,electricity,458.0,,0.2728,124.94
+2011-08-01,electricity,539.0,,0.2728,147.04
+2011-09-01,electricity,391.0,,0.2728,106.66
+2011-10-01,electricity,14.0,,0.2728,3.82
+2011-10-01,electricity,444.0,,0.2698,119.79
+2011-11-01,electricity,2.0,,0.2728,0.54
+2011-11-01,electricity,537.0,,0.2698,144.88
+2011-12-01,electricity,482.0,,0.2698,130.04
+2012-01-01,electricity,-33.0,,0.2698,-8.9
+2012-01-01,electricity,450.0,,0.2759,124.16
+2012-02-01,electricity,421.0,,0.2759,116.15
+2012-03-01,electricity,526.0,,0.2759,145.13
+2012-04-01,electricity,16.0,,0.2759,4.41
+2012-04-01,electricity,482.0,,0.2878,138.72
+2012-05-01,electricity,-2.0,,0.2759,-0.55
+2012-05-01,electricity,385.0,,0.2878,110.8
+2012-06-01,electricity,448.0,,0.2878,128.93
+2011-05-01,gas,18.0,,0.1799,3.24
+2011-06-01,gas,15.0,,0.1799,2.7
+2011-07-01,gas,34.0,,0.1799,6.12
+2011-08-01,gas,2.0,,0.1799,0.36
+2011-08-01,gas,25.0,,0.1961,4.9
+2011-09-01,gas,-1.0,,0.1799,-0.18
+2011-09-01,gas,-2.0,,0.1961,-0.39
+2011-10-01,gas,12.0,,0.1961,2.35
+2011-11-01,gas,12.0,,0.2117,2.54
+2011-12-01,gas,12.0,,0.2117,2.54
+2012-01-01,gas,12.0,,0.2117,2.54
+2012-02-01,gas,11.0,,0.2145,2.36
+2012-03-01,gas,25.0,,0.2145,5.36
+2012-04-01,gas,19.0,,0.2145,4.08
+2012-05-01,gas,-4.0,,0.2145,-0.86
+2012-05-01,gas,15.0,,0.2216,3.32
+2012-06-01,gas,15.0,,0.2216,3.32
+2011-05-01,water,,36.1,1.17,42.24
+2011-05-01,water,,-3.0,1.4,-4.2
+2011-06-01,water,,38.7,1.17,45.28
+2011-07-01,water,,35.0,1.17,40.95
+2011-08-01,water,,40.0,1.17,46.8
+2011-08-01,water,,1.2,1.4,1.68
+2011-09-01,water,,-3.5,1.17,-4.1
+2011-09-01,water,,-1.2,1.4,-1.68
+2011-10-01,water,,8.4,1.17,9.83
+2011-11-01,water,,11.4,1.17,13.34
+2011-12-01,water,,9.6,1.17,11.23
+2012-01-01,water,,12.2,1.17,14.28
+2012-02-01,water,,10.2,1.17,11.93
+2012-03-01,water,,8.3,1.17,9.72
+2012-04-01,water,,9.7,1.17,11.35
+2012-05-01,water,,10.6,1.17,12.4
+2012-06-01,water,,10.3,1.17,12.05

data/scripts/data/elec_and_water_only.csv.sample

@@ -0,0 +1,53 @@
+invoice_month,measure,kwh,cubic_m,rate,amount
+2011-05-01,total_charges,,,,251.44
+2011-06-01,total_charges,,,,235.7
+2011-07-01,total_charges,,,,220.05
+2011-08-01,total_charges,,,,259.2
+2011-09-01,total_charges,,,,119.97
+2011-10-01,total_charges,,,,168.86
+2011-11-01,total_charges,,,,196.46
+2011-12-01,total_charges,,,,176.54
+2012-01-01,total_charges,,,,165.74
+2012-02-01,total_charges,,,,162.64
+2012-03-01,total_charges,,,,193.2
+2012-04-01,total_charges,,,,192.48
+2012-05-01,total_charges,,,,157.31
+2012-06-01,total_charges,,,,177.63
+2011-05-01,electricity,4.0,,0.241,0.97
+2011-05-01,electricity,616.0,,0.2558,157.57
+2011-06-01,electricity,519.0,,0.2558,132.76
+2011-07-01,electricity,-14.0,,0.2558,-3.58
+2011-07-01,electricity,458.0,,0.2728,124.94
+2011-08-01,electricity,539.0,,0.2728,147.04
+2011-09-01,electricity,391.0,,0.2728,106.66
+2011-10-01,electricity,14.0,,0.2728,3.82
+2011-10-01,electricity,444.0,,0.2698,119.79
+2011-11-01,electricity,2.0,,0.2728,0.54
+2011-11-01,electricity,537.0,,0.2698,144.88
+2011-12-01,electricity,482.0,,0.2698,130.04
+2012-01-01,electricity,-33.0,,0.2698,-8.9
+2012-01-01,electricity,450.0,,0.2759,124.16
+2012-02-01,electricity,421.0,,0.2759,116.15
+2012-03-01,electricity,526.0,,0.2759,145.13
+2012-04-01,electricity,16.0,,0.2759,4.41
+2012-04-01,electricity,482.0,,0.2878,138.72
+2012-05-01,electricity,-2.0,,0.2759,-0.55
+2012-05-01,electricity,385.0,,0.2878,110.8
+2012-06-01,electricity,448.0,,0.2878,128.93
+2011-05-01,water,,36.1,1.17,42.24
+2011-05-01,water,,-3.0,1.4,-4.2
+2011-06-01,water,,38.7,1.17,45.28
+2011-07-01,water,,35.0,1.17,40.95
+2011-08-01,water,,40.0,1.17,46.8
+2011-08-01,water,,1.2,1.4,1.68
+2011-09-01,water,,-3.5,1.17,-4.1
+2011-09-01,water,,-1.2,1.4,-1.68
+2011-10-01,water,,8.4,1.17,9.83
+2011-11-01,water,,11.4,1.17,13.34
+2011-12-01,water,,9.6,1.17,11.23
+2012-01-01,water,,12.2,1.17,14.28
+2012-02-01,water,,10.2,1.17,11.93
+2012-03-01,water,,8.3,1.17,9.72
+2012-04-01,water,,9.7,1.17,11.35
+2012-05-01,water,,10.6,1.17,12.4
+2012-06-01,water,,10.3,1.17,12.05

data/scripts/full_analysis.R

@@ -0,0 +1,191 @@
+#!/usr/bin/Rscript
+#
+# This is an example R script that generates a single-page PDF summary.
+#
+# It requires two packages to be installed. Do this from the R command line:
+# install.packages('gplots','zoo')
+#
+# Some sample data and example out is provided:
+#
+# data/all_services.csv.sample        - sample CSV data for a years worth of elec, gas, and water
+# data/all_services.sample.pdf        - PDF analysis produced by this script for all_services.csv.sample
+# data/elec_and_water_only.csv.sample - sample CSV data for a years worth of elec and water
+# data/elec_and_water_only.sample.pdf - PDF analysis produced by this script for elec_and_water_only.csv.sample
+#
+# Usage:
+#   ./full_analysis.R data_file.csv
+#
+# Output is written to a file called 'full_analysis.pdf' in the current directory
+#
+# data_file.csv has the collowing columns (with a header row):
+#   invoice_month,measure,kwh,cubic_m,rate,amount
+#
+# NB: the script scan_all_bills.sh is an example of how to produce a suitable file
+#
+
+# data_file <-'data/all_services.csv.sample'
+argv <- commandArgs(TRUE)
+data_file <- argv[1]
+output_file <- 'full_analysis.pdf'
+
+cat("Analyzing SP Service billing data from:",data_file,"\n")
+cat("Results are written to:",output_file,"\n\n")
+
+# required libs
+library(gplots)
+library(zoo)
+
+#reads: invoice_month,measure,kwh,cubic_m,rate,amount
+bills <- read.csv(data_file)
+
+# core data
+charges <- subset(bills, subset=(measure == 'total_charges'), select = c(-measure,-kwh,-cubic_m,-rate))
+electricity <- subset(bills, subset=(measure == 'electricity'), select = c(-measure,-cubic_m))
+gas <- subset(bills, subset=(measure == 'gas'), select = c(-measure,-cubic_m))
+water <- subset(bills, subset=(measure == 'water'), select = c(-measure,-kwh))
+
+# moving averages over 3 months
+ma_over <- 3
+
+# summarise electricity usage
+#
+electricity.raw.kwh_by_month <- tapply(electricity$kwh, electricity$invoice_month, sum)
+electricity.raw.amount_by_month <- tapply(electricity$amount, electricity$invoice_month, sum)
+
+kwh <- zoo(
+  as.vector(electricity.raw.kwh_by_month),
+  as.Date(unlist(dimnames(electricity.raw.kwh_by_month)))
+)
+amount <- zoo(
+  as.vector(electricity.raw.amount_by_month),
+  as.Date(unlist(dimnames(electricity.raw.amount_by_month)))
+)
+meter <- merge(kwh,amount)
+ma <- rollmean(meter, ma_over, align="center")
+electricity.timeseries <- merge(meter,ma)
+
+# summarise gas usage
+#
+gas.raw.kwh_by_month <- tapply(gas$kwh, gas$invoice_month, sum)
+gas.raw.amount_by_month <- tapply(gas$amount, gas$invoice_month, sum)
+
+kwh <- zoo(
+  as.vector(gas.raw.kwh_by_month),
+  as.Date(unlist(dimnames(gas.raw.kwh_by_month)))
+)
+amount <- zoo(
+  as.vector(gas.raw.amount_by_month),
+  as.Date(unlist(dimnames(gas.raw.amount_by_month)))
+)
+meter <- merge(kwh,amount)
+ma <- rollmean(meter, ma_over, align="center")
+gas.timeseries <- merge(meter,ma)
+
+
+# summarise water usage
+#
+water.raw.cubic_m_by_month <- tapply(water$cubic_m, water$invoice_month, sum)
+water.raw.amount_by_month <- tapply(water$amount, water$invoice_month, sum)
+
+cubic_m <- zoo(
+  as.vector(water.raw.cubic_m_by_month),
+  as.Date(unlist(dimnames(water.raw.cubic_m_by_month)))
+)
+amount <- zoo(
+  as.vector(water.raw.amount_by_month),
+  as.Date(unlist(dimnames(water.raw.amount_by_month)))
+)
+meter <- merge(cubic_m,amount)
+ma <- rollmean(meter, ma_over, align="center")
+water.timeseries <- merge(meter,ma)
+
+# summarise charges
+#
+charges.raw.amount_by_month <- tapply(charges$amount, charges$invoice_month, sum)
+
+amount <- zoo(
+  as.vector(charges.raw.amount_by_month),
+  as.Date(unlist(dimnames(charges.raw.amount_by_month)))
+)
+ma <- rollmean(amount, ma_over, align="center")
+charges.timeseries <- merge(amount,ma)
+
+# combined timeseries
+ts <- merge(charges.timeseries,gas.timeseries,electricity.timeseries,water.timeseries)
+
+
+# begin output
+pdf(output_file)
+
+# split the page into regions for plotting
+par(mfrow=c(3,2))
+
+# print charge summary
+charge_summary <- subset(ts, select=c(amount,amount.meter.electricity.timeseries,amount.meter.gas.timeseries,amount.meter.water.timeseries))
+colnames(charge_summary) <- c('Total','Electricity','Gas','Water')
+textplot(
+  capture.output(summary(as.matrix(charge_summary))),
+  valign="top", halign="center", mar=c(4, 1, 4, 1) + 0.1
+)
+title("Total Charges SGD$")
+
+# plot charge summary
+amount <- as.vector(charges.timeseries$amount)
+bp <- barplot2(amount,
+  main="Total Monthly Bill",
+  names.arg=index(charges.timeseries),
+  ylab="Invoice Amount (SGD$)")
+lines(bp, as.vector(charges.timeseries$ma), lty="dashed", lwd=2)
+
+# alternative plot elec usage summary - basic bar chart
+# barplot(electricity.timeseries$kwh.meter,
+#   main="Electricity Usage",
+#   names.arg=index(electricity.timeseries),
+#   ylab="kwh")
+
+# plot elec usage summary - bar chart with moving average
+kwh <- as.vector(electricity.timeseries$kwh.meter)
+colors <- colorpanel(length(kwh),'red','yellow')
+bp <- barplot2(kwh,
+  main="Electricity Usage",
+  names.arg=index(electricity.timeseries),
+  col=colors,
+  ylab="kwh")
+lines(bp, as.vector(electricity.timeseries$kwh.ma), lty="dashed", lwd=2)
+legend('bottomleft', inset=0.05, legend=c("3m MA"), lty=c("dashed"), bty='o', bg='white')
+
+# plot elec charges summary - line chart with moving average
+df <- subset(electricity.timeseries, select=c(amount.meter,amount.ma))
+x <- index(electricity.timeseries)
+plot(df, screens=1,
+  lty=c("solid", "dashed"), lwd=c(1,2), col=c('red','black'),
+  main="Electricity Charges", xlab='', ylab="SGD$",
+  xaxt = 'n')
+axis(1, x, x)
+legend('bottomleft', legend=c("meter","3m MA"), lty=c("solid", "dashed"), col=c('red','black'), bty='n')
+
+# plot gas summary
+kwh <- as.vector(gas.timeseries$kwh.meter)
+kwh[is.na(kwh)] <- 0 # in case no values
+colors <- colorpanel(length(kwh),'#0014EB','#0077EB')
+bp <- barplot2(kwh,
+  main="Gas Usage",
+  names.arg=index(gas.timeseries),
+  col=colors,
+  ylab="kwh")
+lines(bp, as.vector(gas.timeseries$kwh.ma), lty="dashed", lwd=2)
+
+# plot water summary
+cubic_m <- as.vector(water.timeseries$cubic_m.meter)
+colors <- colorpanel(length(cubic_m),'#00EBEB','#009D62')
+bp <- barplot2(cubic_m,
+  main="Water Usage",
+  names.arg=index(water.timeseries),
+  col=colors,
+  ylab="m^3")
+lines(bp, as.vector(water.timeseries$cubic_m.ma), lty="dashed", lwd=2)
+
+
+cat("Completed analyzing SP Service billing data from:",data_file,"\n")
+cat("Results are written to:",output_file,"\n\n")
+

data/spec/support/bill_examples.rb

@@ -5,7 +5,7 @@ shared_examples_for "has a valid reader" do |resource_key|
     let(:resource) { eval "#{resource_key}" }
     let(:reader) { resource.reader }
     subject { reader }
-    it { should be_a(PDF::StructuredReader) }
+    it { should be_a(PDF::Reader::Turtletext) }
   end
 end
 

data/spec/support/pdf_samples_helper.rb

@@ -20,11 +20,11 @@ module PdfSamplesHelper
   end
 
   def personal_pdf_sample_names
-    Dir["#{File.dirname(__FILE__)}/../fixtures/personal_pdf_samples/*.pdf"]
+    Dir[personal_pdf_sample_path.join("*.pdf")]
   end
 
   def personal_pdf_sample_expectations_path
-    Pathname.new(File.dirname(__FILE__)).join('..','fixtures','personal_pdf_samples','expectations.yml')
+    Pathname.new(personal_pdf_sample_path.join('expectations.yml'))
   end
 
   def personal_pdf_sample_expectations

data/spec/unit/bill_collection_spec.rb

@@ -96,9 +96,9 @@ describe SpsBill::BillCollection do
     let(:bills) { SpsBill::BillCollection.new }
     before do
       bill_a.instance_variable_set(:@invoice_month, Date.parse('2012-02-01'))
-      bill_a.instance_variable_set(:@electricity_usage, [{ kwh: 12.0, rate: 0.1234, amount: 12.34 }] )
+      bill_a.instance_variable_set(:@electricity_usage, [{ :kwh => 12.0, :rate => 0.1234, :amount => 12.34 }] )
       bill_b.instance_variable_set(:@invoice_month, Date.parse('2012-03-01'))
-      bill_b.instance_variable_set(:@electricity_usage, [{ kwh: 24.0, rate: 0.5678, amount: 56.78 }])
+      bill_b.instance_variable_set(:@electricity_usage, [{ :kwh => 24.0, :rate => 0.5678, :amount => 56.78 }])
       bills << bill_a
       bills << bill_b
     end
@@ -122,9 +122,9 @@ describe SpsBill::BillCollection do
     let(:bills) { SpsBill::BillCollection.new }
     before do
       bill_a.instance_variable_set(:@invoice_month, Date.parse('2012-02-01'))
-      bill_a.instance_variable_set(:@gas_usage, [{ kwh: 12.0, rate: 0.1234, amount: 12.34 }] )
+      bill_a.instance_variable_set(:@gas_usage, [{ :kwh => 12.0, :rate => 0.1234, :amount => 12.34 }] )
       bill_b.instance_variable_set(:@invoice_month, Date.parse('2012-03-01'))
-      bill_b.instance_variable_set(:@gas_usage, [{ kwh: 24.0, rate: 0.5678, amount: 56.78 }])
+      bill_b.instance_variable_set(:@gas_usage, [{ :kwh => 24.0, :rate => 0.5678, :amount => 56.78 }])
       bills << bill_a
       bills << bill_b
     end
@@ -147,9 +147,9 @@ describe SpsBill::BillCollection do
     let(:bills) { SpsBill::BillCollection.new }
     before do
       bill_a.instance_variable_set(:@invoice_month, Date.parse('2012-02-01'))
-      bill_a.instance_variable_set(:@water_usage, [{ cubic_m: 12.0, rate: 0.1234, amount: 12.34 }] )
+      bill_a.instance_variable_set(:@water_usage, [{ :cubic_m => 12.0, :rate => 0.1234, :amount => 12.34 }] )
       bill_b.instance_variable_set(:@invoice_month, Date.parse('2012-03-01'))
-      bill_b.instance_variable_set(:@water_usage, [{ cubic_m: 24.0, rate: 0.5678, amount: 56.78 }])
+      bill_b.instance_variable_set(:@water_usage, [{ :cubic_m => 24.0, :rate => 0.5678, :amount => 56.78 }])
       bills << bill_a
       bills << bill_b
     end

data/spec/unit/shell_spec.rb

@@ -6,7 +6,7 @@ describe SpsBill::Shell do
   describe "##usage" do
     subject { SpsBill::Shell.usage }
     it "should print help" do
-      STDOUT.should_receive(:puts)
+      STDOUT.should_receive(:puts) if RUBY_ENGINE.to_s != 'jruby' # cannot get this to work with JRuby yet
       subject
     end
   end

data/sps_bill.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "sps_bill"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Paul Gallagher"]
-  s.date = "2012-07-20"
+  s.date = "2012-08-01"
   s.description = "a library that can read SP Services PDF bills and extract and summarize the bill details"
   s.email = "gallagher.paul@gmail.com"
   s.executables = ["sps_bill"]
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
     ".rspec",
     ".rvmrc",
     ".travis.yml",
+    "CHANGELOG",
     "Gemfile",
     "Gemfile.lock",
     "Guardfile",
@@ -29,10 +30,6 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "bin/sps_bill",
-    "lib/pdf/object_hash.rb",
-    "lib/pdf/positional_text_receiver.rb",
-    "lib/pdf/structured_reader.rb",
-    "lib/pdf/textangle.rb",
     "lib/sps_bill.rb",
     "lib/sps_bill/bill.rb",
     "lib/sps_bill/bill_collection.rb",
@@ -40,9 +37,13 @@ Gem::Specification.new do |s|
     "lib/sps_bill/shell.rb",
     "lib/sps_bill/version.rb",
     "scripts/data/.gitkeep",
+    "scripts/data/all_services.csv.sample",
+    "scripts/data/all_services.sample.pdf",
+    "scripts/data/elec_and_water_only.csv.sample",
+    "scripts/data/elec_and_water_only.sample.pdf",
+    "scripts/full_analysis.R",
     "scripts/scan_all_bills.sh",
     "spec/fixtures/pdf_samples/.gitkeep",
-    "spec/fixtures/pdf_samples/junk_prefix.pdf",
     "spec/fixtures/personal_pdf_samples/.gitkeep",
     "spec/fixtures/personal_pdf_samples/expectations.yml.sample",
     "spec/integration/personal_samples_spec.rb",
@@ -51,7 +52,6 @@ Gem::Specification.new do |s|
     "spec/support/pdf_samples_helper.rb",
     "spec/unit/bill_collection_spec.rb",
     "spec/unit/bill_spec.rb",
-    "spec/unit/pdf/object_hash_spec.rb",
     "spec/unit/shell_spec.rb",
     "sps_bill.gemspec"
   ]
@@ -65,32 +65,29 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<pdf-reader>, ["= 1.1.1"])
+      s.add_runtime_dependency(%q<pdf-reader-turtletext>, ["~> 0.2.2"])
       s.add_runtime_dependency(%q<getoptions>, ["~> 0.3"])
       s.add_development_dependency(%q<bundler>, ["~> 1.1.4"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_development_dependency(%q<rcov>, [">= 0"])
       s.add_development_dependency(%q<rake>, ["~> 0.9.2.2"])
       s.add_development_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.11"])
       s.add_development_dependency(%q<guard-rspec>, [">= 0"])
     else
-      s.add_dependency(%q<pdf-reader>, ["= 1.1.1"])
+      s.add_dependency(%q<pdf-reader-turtletext>, ["~> 0.2.2"])
       s.add_dependency(%q<getoptions>, ["~> 0.3"])
       s.add_dependency(%q<bundler>, ["~> 1.1.4"])
       s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-      s.add_dependency(%q<rcov>, [">= 0"])
       s.add_dependency(%q<rake>, ["~> 0.9.2.2"])
       s.add_dependency(%q<rspec>, ["~> 2.8.0"])
       s.add_dependency(%q<rdoc>, ["~> 3.11"])
       s.add_dependency(%q<guard-rspec>, [">= 0"])
     end
   else
-    s.add_dependency(%q<pdf-reader>, ["= 1.1.1"])
+    s.add_dependency(%q<pdf-reader-turtletext>, ["~> 0.2.2"])
     s.add_dependency(%q<getoptions>, ["~> 0.3"])
     s.add_dependency(%q<bundler>, ["~> 1.1.4"])
     s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
-    s.add_dependency(%q<rcov>, [">= 0"])
     s.add_dependency(%q<rake>, ["~> 0.9.2.2"])
     s.add_dependency(%q<rspec>, ["~> 2.8.0"])
     s.add_dependency(%q<rdoc>, ["~> 3.11"])