pdf-extract 0.0.10 → 0.1.0

data/lib/font_metrics.rb

@@ -29,7 +29,7 @@ module PdfExtract
       @ascent = 0
       @descent = 0
       @bbox = [0, 0, 0, 0]
-      
+
       base_font = font.basefont.to_s
       if @@base_fonts.key? base_font
         @ascent = @@base_fonts[base_font][:Ascent]
@@ -37,12 +37,20 @@ module PdfExtract
         @bbox = @@base_fonts[base_font][:FontBBox]
         @glyph_width_lookup = proc { |c|
           @@base_fonts[base_font][:Widths].fetch(c.codepoints.first, 0)
-        } 
+        }
       else
         @ascent = font.ascent
         @descent = font.descent
         @bbox = font.bbox
-        @glyph_width_lookup = proc { |c| font.glyph_width c }
+        @glyph_width_lookup = proc do |c|
+          begin
+            font.glyph_width c.codepoints.first
+          rescue TypeError => e
+            # It seems some fonts don't have a first char attribute in their
+            # descriptor and this causes problems for pdf-reader.
+            0
+          end
+        end
       end
 
       if not @bbox.nil?

data/lib/language.rb

@@ -17,7 +17,7 @@ module PdfExtract::Language
       when "\ufb05" then r << "ft"
       when "\ufb06" then r << "st"
       when "\u1d6b" then r << "ue"
-      
+
       # Normalise some punctuation.
       when "\u2018" then r << "'"
       when "\u2019" then r << "'"
@@ -33,19 +33,19 @@ module PdfExtract::Language
         r << c
       end
     end
-      
+
     r.gsub /\s+/, " "
   end
-  
+
   def self.letter_ratio s
     s.count("A-Z0-9\-[],.\"'()") / s.length.to_f
   end
-  
+
   # TODO Ignore caps in middle of words
   def self.cap_ratio s
     sentence_end = true
     cap_count = 0
-    
+
     s.each_char do |c|
       if c =~ /\./
         sentence_end = true
@@ -56,13 +56,13 @@ module PdfExtract::Language
         sentence_end = false
       end
     end
-    
+
     cap_count / s.split.length.to_f
   end
-  
+
   def self.year_ratio s
     words = s.split
-    
+
     year_words = words.map do |word|
       word =~ /[^\d]\d{4}[^\d]/
     end
@@ -77,6 +77,6 @@ module PdfExtract::Language
   def self.word_count s
     s.split.count
   end
-  
+
 end
 

data/lib/model/chunks.rb

@@ -24,14 +24,14 @@ module PdfExtract
     }
 
     def self.include_in pdf
-      
+
       pdf.spatials :chunks, :paged => true, :depends_on => [:characters] do |parser|
         rows = {}
 
         parser.before do
           rows = {}
         end
-        
+
         parser.objects :characters do |chars|
           y = chars[:y]
           rows[y] = [] if rows[y].nil?
@@ -48,7 +48,7 @@ module PdfExtract
           char_slop = pdf.settings[:char_slop]
           word_slop = pdf.settings[:word_slop]
           overlap_slop = pdf.settings[:overlap_slop]
-          
+
           text_chunks = []
 
           rows.each_pair do |y, row|
@@ -105,7 +105,11 @@ module PdfExtract
           end
 
           merged_text_chunks << text_chunks.first
-        end 
+
+          # Remove empty lines - they mess up region detection by
+          # making them join together.
+          merged_text_chunks.reject { |chunk| chunk[:content].strip == "" }
+        end
       end
     end
 

data/lib/model/regions.rb

@@ -41,7 +41,7 @@ to be part of the same region. :line_slop is multiplied by the average line heig
         height_taken = from_top + line[:height]
       end
     end
-    
+
     def self.include_in pdf
       pdf.spatials :regions, :paged => true, :depends_on => [:chunks] do |parser|
         chunks = []
@@ -51,7 +51,7 @@ to be part of the same region. :line_slop is multiplied by the average line heig
           chunks = []
           regions = []
         end
-        
+
         parser.objects :chunks do |chunk|
           y = chunk[:y].floor
 
@@ -72,16 +72,16 @@ to be part of the same region. :line_slop is multiplied by the average line heig
             chunk[:lines] = [Spatial.as_line(chunk)]
             chunk.delete :content
           end
-          
+
           compare_index = 1
           while chunks.count > compare_index
             b = chunks.first
             t = chunks[compare_index]
-              
+
             line_height = b[:line_height]
             line_slop = [line_height, t[:height]].min * pdf.settings[:line_slop]
             incident_y = (b[:y] + b[:height] + line_slop) >= t[:y]
-            
+
             if incident_y && incident(t, b)
               chunks[0] = Spatial.merge t, b, :lines => true
               chunks.delete_at compare_index
@@ -96,7 +96,7 @@ to be part of the same region. :line_slop is multiplied by the average line heig
               compare_index = 1
             end
           end
-          
+
           regions << chunks.first unless chunks.first.nil?
 
           regions.each do |region|
@@ -112,6 +112,6 @@ to be part of the same region. :line_slop is multiplied by the average line heig
         end
       end
     end
-    
+
   end
 end

data/lib/multi_range.rb

@@ -15,7 +15,7 @@ module PdfExtract
         r.include?(range.min) || r.include?(range.max) ||
           range.include?(r.min) || range.include?(r.max)
       end
-      
+
       incident << range
 
       non_incident = @ranges - incident
@@ -46,11 +46,21 @@ module PdfExtract
     end
 
     def max
-      @max ||= @ranges.sort_by { |r| -r.max }.first.max
+      @ranges.sort_by { |r| -r.max }.first.max
     end
 
     def min
-      @min ||= @ranges.sort_by { |r| r.min }.first.min
+      @ranges.sort_by { |r| r.min }.first.min
+    end
+
+    def widest
+      widest = @ranges.sort_by { |r| r.max - r.min }.last
+      widest.max - widest.min
+    end
+
+    def narrowest
+      narrowest = @ranges.sort_by { |r| r.max - r.min }.first
+      narrowest.max - narrowest.min
     end
 
     def avg

data/lib/pdf-extract.rb

@@ -9,7 +9,6 @@ require_relative 'analysis/columns'
 require_relative 'analysis/sections'
 require_relative 'references/references'
 require_relative 'references/resolved_references'
-require_relative 'view/png_view'
 require_relative 'view/pdf_view'
 require_relative 'view/xml_view'
 
@@ -68,7 +67,6 @@ module PdfExtract
     add_parser ResolvedReferences
 
     add_view :pdf, PdfView
-    add_view :png, PngView
     add_view :xml, XmlView
   end
 

data/lib/references/references.rb

@@ -6,7 +6,7 @@ module PdfExtract
   module References
 
     Settings.declare :reference_flex, {
-      :default => 0.1,
+      :default => 0.2,
       :module => self.name,
       :description => "Article sections are given a score as potential reference sections. Their score is based on article section features, such as the number of family names that appear, the ratio of uppercase letters to lowercase, and so on. Any article section that has a score that is more than 1 - :reference_flex percent of the best score will be parsed as a reference section."
     }
@@ -16,7 +16,7 @@ module PdfExtract
       :module => self.name,
       :description => "There must be :min_sequence_count or more numbered references within a candidate reference section for them to be parsed as number-delimited references."
     }
-    
+
     Settings.declare :max_reference_order, {
       :default => 1000,
       :module => self.name,
@@ -82,11 +82,11 @@ module PdfExtract
 
       # Determine the charcaters that are most likely part of numeric
       # delimiters.
-      
+
       after = {}
       before = {}
       last_n = -1
-      
+
       s.scan /[^\d]?\d+[^\d]/ do |m|
         n = m[/\d+/].to_i
         if n < pdf.settings[:max_reference_order]
@@ -115,14 +115,14 @@ module PdfExtract
       if ["", "\\[", "\\ "].include?(b_s) && ["", "\\.", "\\]", "\\ "].include?(a_s)
 
         # Split by the delimiters and record separate refs.
-        
+
         last_n = -1
         current_ref = ""
         refs = []
         parts = s.partition(Regexp.new "#{b_s}?\\d+#{a_s}")
-        
+
         while not parts[1].length.zero?
-          n = parts[1][/\d+/].to_i          
+          n = parts[1][/\d+/].to_i
           if n < pdf.settings[:max_reference_order] && last_n == -1
             last_n = n
           elsif n == last_n.next
@@ -139,12 +139,12 @@ module PdfExtract
 
           parts = parts[2].partition(Regexp.new "#{b_s}?\\d+#{a_s}")
         end
-        
+
         refs << {
           :content => (current_ref + parts[0]).strip,
           :order => last_n
         }
-        
+
         refs
 
       else
@@ -177,7 +177,7 @@ module PdfExtract
 
       seq_count >= pdf.settings[:min_sequence_count]
     end
-    
+
     def self.include_in pdf
       pdf.spatials :references, :depends_on => [:sections] do |parser|
 
@@ -190,7 +190,7 @@ module PdfExtract
         parser.after do
           max_score = sections.map {|s| s[:reference_score]}.max
           min_permittable = max_score - (max_score * pdf.settings[:reference_flex])
-          
+
           refs = []
 
           sections = sections.reject do |s|
@@ -199,13 +199,14 @@ module PdfExtract
             # half of an article.
             s[:lateness] < pdf.settings[:min_lateness] || s[:year_ratio].zero?
           end
-          
+
           sections.each do |section|
             if section[:reference_score] >= min_permittable
             # TODO Enable classification once we have a reasonable model.
             #if Score.reference?(section)
-              if numeric_sequence? pdf, Spatial.get_text_content(section)
-                refs += split_by_delimiter pdf, Spatial.get_text_content(section)
+              content = Spatial.get_text_content(section)
+              if numeric_sequence? pdf, content
+                refs += split_by_delimiter pdf, content
               elsif multi_margin? section[:lines]
                 refs += split_by_margin section[:lines]
               elsif multi_spacing? section[:lines]
@@ -213,7 +214,7 @@ module PdfExtract
               end
             end
           end
-          
+
           # TODO Ideally we wouldn't see the ref headers here.
           # Unfortunately publication details can look a lot like references.
           refs.reject do |ref|

data/lib/references/resolve.rb

@@ -12,12 +12,12 @@ module PdfExtract::Resolve
       resolved = {}
       begin
         doc = Nokogiri::HTML(open url)
-      
+
         result = doc.at_css "div.result"
         unless result.nil?
           score = result.at_css("span.cr_score").content.to_s
           if score.to_i >= 90
-            doi = result.at_css "span.doi" 
+            doi = result.at_css "span.doi"
             resolved[:doi] = doi.content.sub "http://dx.doi.org/", ""
           end
         end
@@ -25,17 +25,17 @@ module PdfExtract::Resolve
       end
       resolved
     end
-    
+
   end
-  
+
   class FreeCite
-    
+
     def self.find ref
       Net::HTTP.start "freecite.library.brown.edu" do |http|
         r = http.post "/citations/create", "citation=#{ref}",
                       "Accept" => "text/xml"
         doc = Nokogiri::XML r.body
-        
+
         {
           :title => doc.at_xpath("//title").content,
           :journal => doc.at_xpath("//journal").content,
@@ -44,13 +44,13 @@ module PdfExtract::Resolve
         }
       end
     end
-    
+
   end
-  
+
   class SimpleTextQuery
 
     @@cookie = nil
-    
+
     def self.find ref
       create_session
 
@@ -68,10 +68,10 @@ module PdfExtract::Resolve
       response = Net::HTTP.start "www.crossref.org" do |http|
         http.request post
       end
-      
+
       doc = Nokogiri::HTML response.body
       doi = doc.at_css "td.resultB > a"
-      
+
       if doi.nil?
         {}
       else
@@ -87,11 +87,11 @@ module PdfExtract::Resolve
         end
       end
     end
-    
+
   end
-  
+
   @@resolvers = [Sigg]
-  
+
   def self.resolvers= resolver
     @@resolvers = resolver
   end
@@ -109,5 +109,5 @@ module PdfExtract::Resolve
     end
     ref
   end
-  
+
 end

data/lib/references/score.rb

@@ -8,7 +8,7 @@ module PdfExtract
     end
 
     @@reference_model = Model.new(path_to_data("reference.model"))
-    
+
     def self.reference? section
       sample = {
         1 => section[:letter_ratio],

data/lib/spatial.rb

@@ -24,13 +24,13 @@ module PdfExtract
 
     def self.merge_lines a, b, so
       so[:lines] = []
-      
+
       if a.key? :lines
         so[:lines] += a[:lines]
       else
         so[:lines] << as_line(a)
       end
-      
+
       if b.key? :lines
         so[:lines] += b[:lines]
       else
@@ -60,7 +60,7 @@ module PdfExtract
         so[:content] = (a[:content] + options[:separator] + b[:content])
         so[:content] = so[:content].gsub /\s+/, " "
       end
-      
+
       if get_text_content(a).length > get_text_content(b).length
         so[:font] = a[:font]
         so[:line_height] = a[:line_height]
@@ -115,12 +115,12 @@ module PdfExtract
     # correct write order, specified by write_mode.
     def self.collapse objs, options={}
       options = @@default_options.merge options
-      
+
       sorted = case write_mode
                when :left_to_right
                  objs.sort_by { |obj| -(obj[:y].floor * 100) + (obj[:x] / 100.0) }
                end
-      
+
       if sorted.count == 1
         sorted.first.dup
       else
@@ -132,18 +132,18 @@ module PdfExtract
       end
     end
 
-    def self.contains? a, b
-      a_x1 = a[:x]
-      a_x2 = a[:x] + a[:width]
-      a_y1 = a[:y]
-      a_y2 = a[:y] + a[:height]
+    def self.contains? a, b, padding=0
+      a_x1 = a[:x] - padding
+      a_x2 = a[:x] + a[:width] + (padding * 2)
+      a_y1 = a[:y] - padding
+      a_y2 = a[:y] + a[:height] + (padding * 2)
 
       b_x1 = b[:x]
       b_x2 = b[:x] + b[:width]
       b_y1 = b[:y]
       b_y2 = b[:y] + b[:height]
 
-      b_x1 >= a_x1 && b_x2 <= a_x2 && b_y1 >= a_y1 && b_y2 <= a_y2 
+      b_x1 >= a_x1 && b_x2 <= a_x2 && b_y1 >= a_y1 && b_y2 <= a_y2
     end
 
     def self.overlap? from, by, a, b
@@ -158,7 +158,7 @@ module PdfExtract
         diffs = items.map {|item| (item[f] - ideals[f][0]).abs}
         diffs.map! {|d| d.nan? ? 1 : d}
         max_diff = diffs.max
-        
+
         scores = diffs.map do |d|
           if d == 0
             ideals[f][1]
@@ -173,6 +173,6 @@ module PdfExtract
         end
       end
     end
-    
+
   end
 end