sps_bill 0.1.0 → 0.1.1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sps_bill
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-20 00:00:00.000000000 Z
+date: 2012-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pdf-reader
-  requirement: &70313136422520 !ruby/object:Gem::Requirement
+  name: pdf-reader-turtletext
+  requirement: &70331993763820 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - =
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1.1
+        version: 0.2.2
   type: :runtime
   prerelease: false
-  version_requirements: *70313136422520
+  version_requirements: *70331993763820
 - !ruby/object:Gem::Dependency
   name: getoptions
-  requirement: &70313136421900 !ruby/object:Gem::Requirement
+  requirement: &70331993762960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *70313136421900
+  version_requirements: *70331993762960
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70313136421060 !ruby/object:Gem::Requirement
+  requirement: &70331993762360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 1.1.4
   type: :development
   prerelease: false
-  version_requirements: *70313136421060
+  version_requirements: *70331993762360
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &70313136420480 !ruby/object:Gem::Requirement
+  requirement: &70331993761820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,21 +54,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *70313136420480
-- !ruby/object:Gem::Dependency
-  name: rcov
-  requirement: &70313136419920 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: *70313136419920
+  version_requirements: *70331993761820
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70313136419140 !ruby/object:Gem::Requirement
+  requirement: &70331993760540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -76,10 +65,10 @@ dependencies:
         version: 0.9.2.2
   type: :development
   prerelease: false
-  version_requirements: *70313136419140
+  version_requirements: *70331993760540
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70313136418160 !ruby/object:Gem::Requirement
+  requirement: &70331993759960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -87,10 +76,10 @@ dependencies:
         version: 2.8.0
   type: :development
   prerelease: false
-  version_requirements: *70313136418160
+  version_requirements: *70331993759960
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &70313136417580 !ruby/object:Gem::Requirement
+  requirement: &70331993743820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +87,10 @@ dependencies:
         version: '3.11'
   type: :development
   prerelease: false
-  version_requirements: *70313136417580
+  version_requirements: *70331993743820
 - !ruby/object:Gem::Dependency
   name: guard-rspec
-  requirement: &70313136417060 !ruby/object:Gem::Requirement
+  requirement: &70331993743100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +98,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70313136417060
+  version_requirements: *70331993743100
 description: a library that can read SP Services PDF bills and extract and summarize
   the bill details
 email: gallagher.paul@gmail.com
@@ -124,6 +113,7 @@ files:
 - .rspec
 - .rvmrc
 - .travis.yml
+- CHANGELOG
 - Gemfile
 - Gemfile.lock
 - Guardfile
@@ -131,10 +121,6 @@ files:
 - README.rdoc
 - Rakefile
 - bin/sps_bill
-- lib/pdf/object_hash.rb
-- lib/pdf/positional_text_receiver.rb
-- lib/pdf/structured_reader.rb
-- lib/pdf/textangle.rb
 - lib/sps_bill.rb
 - lib/sps_bill/bill.rb
 - lib/sps_bill/bill_collection.rb
@@ -142,9 +128,13 @@ files:
 - lib/sps_bill/shell.rb
 - lib/sps_bill/version.rb
 - scripts/data/.gitkeep
+- scripts/data/all_services.csv.sample
+- scripts/data/all_services.sample.pdf
+- scripts/data/elec_and_water_only.csv.sample
+- scripts/data/elec_and_water_only.sample.pdf
+- scripts/full_analysis.R
 - scripts/scan_all_bills.sh
 - spec/fixtures/pdf_samples/.gitkeep
-- spec/fixtures/pdf_samples/junk_prefix.pdf
 - spec/fixtures/personal_pdf_samples/.gitkeep
 - spec/fixtures/personal_pdf_samples/expectations.yml.sample
 - spec/integration/personal_samples_spec.rb
@@ -153,7 +143,6 @@ files:
 - spec/support/pdf_samples_helper.rb
 - spec/unit/bill_collection_spec.rb
 - spec/unit/bill_spec.rb
-- spec/unit/pdf/object_hash_spec.rb
 - spec/unit/shell_spec.rb
 - sps_bill.gemspec
 homepage: https://github.com/tardate/sps_bill_scanner

data/lib/pdf/object_hash.rb

@@ -1,39 +0,0 @@
-class PDF::Reader
-  class ObjectHash
-
-    def extract_io_from(input)
-      if input.respond_to?(:seek) && input.respond_to?(:read)
-        input
-      elsif File.file?(input.to_s)
-        read_with_quirks(input)
-      else
-        raise ArgumentError, "input must be an IO-like object or a filename"
-      end
-    end
-
-    # Load file as a StringIO stream, accounting for invalid format
-    # where additional characters exist in the file before the %PDF start of file
-    def read_with_quirks(input)
-      stream = File.open(input.to_s, "rb")
-      if ofs = pdf_offset(stream)
-        stream.seek(ofs)
-        StringIO.new(stream.read)
-      else
-        raise ArgumentError, "invalid file format"
-      end
-    end
-    private :read_with_quirks
-
-    # Returns the offset of the PDF document in the +stream+.
-    # Checks up to 50 chars into the file, returns nil of no PDF stream detected.
-    def pdf_offset(stream)
-      stream.rewind
-      ofs = stream.pos
-      until (c = stream.readchar) == '%' || c == 37 || ofs > 50
-        ofs += 1
-      end
-      ofs < 50 ? ofs : nil
-    end
-    private :pdf_offset
-  end
-end

data/lib/pdf/positional_text_receiver.rb

@@ -1,16 +0,0 @@
-class PDF::Reader::PositionalTextReceiver < PDF::Reader::PageTextReceiver
-
-  # record text that is drawn on the page
-  def show_text(string) # Tj
-    raise PDF::Reader::MalformedPDFError, "current font is invalid" if @state.current_font.nil?
-    newx, newy = @state.trm_transform(0,0)
-    @content[newy] ||= {}
-    @content[newy][newx] = @state.current_font.to_utf8(string)
-  end
-
-  # override content accessor
-  def content
-    @content
-  end
-
-end

data/lib/pdf/structured_reader.rb

@@ -1,108 +0,0 @@
-# Class for reading structured text content
-# This is the one that is a bit hairy - specifically check the fuzzed_y usage
-# which attempts to align text content in the PDF so it can be extracted
-# with correct alignment.
-#
-class PDF::StructuredReader
-  attr_reader :reader
-
-  # +source+ is a file name or stream-like object
-  def initialize(source)
-    @reader = PDF::Reader.new(source)
-  end
-
-  # Returns positional (with fuzzed y positioning) text content collection as a hash:
-  # { y_position: { x_position: content}}
-  def content(page=1)
-    @content ||= []
-    if @content[page]
-      @content[page]
-    else
-      @content[page] = fuzzed_y(precise_content(page))
-    end
-  end
-
-  # Returns a hash with fuzzed y positioning:
-  # { fuzzed_y_position: { x_position: content}}
-  # Given +input+ as a hash:
-  # { y_position: { x_position: content}}
-  # y values that fall within +precision+ points of another will be clustered
-  def fuzzed_y(input,precision=3)
-    output = {}
-    input.keys.sort.each do |precise_y|
-      # matching_y = (precise_y / 5.0).truncate * 5.0
-      matching_y = output.keys.select{|new_y| (new_y - precise_y).abs < precision }.first || precise_y
-      output[matching_y] ||= {}
-      output[matching_y].merge!(input[precise_y])
-    end
-    output
-  end
-
-  # Returns positional text content collection as a hash with precise x,y positioning:
-  # { y_position: { x_position: content}}
-  def precise_content(page=1)
-    @precise_content ||= []
-    if @precise_content[page]
-      @precise_content[page]
-    else
-      @precise_content[page] = load_content(page)
-    end
-  end
-
-  # Returns an array of text elements in the bounding box
-  def text_in_rect(xmin,xmax,ymin,ymax,page=1)
-    text_map = content(page)
-    box = []
-    text_map.keys.sort.reverse.each do |y|
-      if y >= ymin && y<= ymax
-        row = []
-        text_map[y].keys.sort.each do |x|
-          if x >= xmin && x<= xmax
-            row << text_map[y][x]
-          end
-        end
-        box << row unless row.empty?
-      end
-    end
-    box
-  end
-
-  # Returns the position {x: val, y: val } of +text+ on +page+
-  # +text+ may be  astring (exact match required) or a Regexp
-  def text_position(text,page=1)
-    item = if text.class <= Regexp
-      content(page).map {|k,v| if x = v.reduce(nil){|memo,vv|  memo = (vv[1] =~ text) ? vv[0] : memo  } ; [k,x] ; end }
-    else
-      content(page).map {|k,v| if x = v.rassoc(text) ; [k,x] ; end }
-    end
-    item = item.compact.flatten
-    unless item.empty?
-      { :x => item[1], :y => item[0] }
-    end
-  end
-
-  # WIP - not using Textangle yet for text extraction.
-  # Ideal usage is something like this:
-  #
-  # textangle = reader.bounding_box do
-  #   page 1
-  #   below "Electricity Services"
-  #   above "Gas Services by City Gas Pte Ltd"
-  #   right_of 240.0
-  #   left_of "Total ($)"
-  # end
-  # textangle.text
-  #
-  def bounding_box(&block)
-    PDF::Reader::Textangle.new(self,&block)
-  end
-
-  private
-
-    def load_content(page)
-      receiver = PDF::Reader::PositionalTextReceiver.new
-      reader.page(page).walk(receiver)
-      receiver.content
-    end
-
-end

data/lib/pdf/textangle.rb

@@ -1,27 +0,0 @@
-# A DSL syntax for text extraction.
-# WIP - not using this yet
-#
-# textangle = PDF::Reader::Textangle.new(reader) do
-#   page 1
-#   below "Electricity Services"
-#   above "Gas Services by City Gas Pte Ltd"
-#   right_of 240.0
-#   left_of "Total ($)"
-# end
-# textangle.text
-#
-class PDF::Reader::Textangle
-  attr_reader :reader
-  attr_writer :page,:above,:below,:left_of,:right_of
-
-  # +structured_reader+ is a PDF::StructuredReader
-  def initialize(structured_reader,&block)
-    @reader = structured_reader
-    instance_eval( &block ) if block
-  end
-
-  def text
-    # TODO
-  end
-
-end

data/spec/fixtures/pdf_samples/junk_prefix.pdf

@@ -1,71 +0,0 @@
-<html>
-<head></head>
-%PDF-1.3
-%����
-1 0 obj
-<< /Creator <feff0050007200610077006e>
-/Producer <feff0050007200610077006e>
->>
-endobj
-2 0 obj
-<< /Type /Catalog
-/Pages 3 0 R
->>
-endobj
-3 0 obj
-<< /Type /Pages
-/Count 1
-/Kids [5 0 R]
->>
-endobj
-4 0 obj
-<< /Length 157
->>
-stream
-q
-
-BT
-36 747.384 Td
-/F1.0 12 Tf
-[<546869732050444620636f6e7461696e73206a756e6b20626566> 30 <6f72652074686520252d504446206d6172> -15 <6b> 20 <6572>] TJ
-ET
-
-Q
-
-endstream
-endobj
-5 0 obj
-<< /Type /Page
-/Parent 3 0 R
-/MediaBox [0 0 612.0 792.0]
-/Contents 4 0 R
-/Resources << /ProcSet [/PDF /Text /ImageB /ImageC /ImageI]
-/Font << /F1.0 6 0 R
->>
->>
->>
-endobj
-6 0 obj
-<< /Type /Font
-/Subtype /Type1
-/BaseFont /Helvetica
-/Encoding /WinAnsiEncoding
->>
-endobj
-xref
-0 7
-0000000000 65535 f 
-0000000015 00000 n 
-0000000109 00000 n 
-0000000158 00000 n 
-0000000215 00000 n 
-0000000423 00000 n 
-0000000601 00000 n 
-trailer
-<< /Size 7
-/Root 2 0 R
-/Info 1 0 R
->>
-startxref
-698
-%%EOF

data/spec/unit/pdf/object_hash_spec.rb

@@ -1,15 +0,0 @@
-require 'spec_helper'
-include PdfSamplesHelper
-
-describe PDF::Reader::ObjectHash do
-
-  context "when there is a junk prefix" do
-    let(:sample_name) { junk_prefix_pdf_sample_name }
-    let(:object_hash) { PDF::Reader::ObjectHash.new(sample_name) }
-    let(:stream) { object_hash.instance_variable_get(:@io) }
-    before { stream.rewind }
-    subject { stream.read(4) }
-    it { should eql("%PDF") }
-  end
-
-end