pdf-extract 0.0.1

data/lib/spatial.rb

@@ -0,0 +1,188 @@
+
+module PdfExtract
+  module Spatial
+
+    @@default_options = {
+      :separator => '',
+      :lines => false,
+      :write_mode => :left_to_right
+    }
+
+    @@spatial_attribs = [:x, :y, :width, :height, :page_width, :page_height, :page]
+
+    def self.concat_lines top, bottom
+      if top =~ /\-\Z/
+        top[0..-2] + bottom
+      else
+        top + ' ' + bottom
+      end
+    end
+
+    def self.drop_spatial obj
+      obj.dup.delete_if { |k, v| @@spatial_attribs.include? k }
+    end
+
+    def self.merge_lines a, b, so
+      so[:lines] = []
+      
+      if a.key? :lines
+        so[:lines] += a[:lines]
+      else
+        so[:lines] << as_line(a)
+      end
+      
+      if b.key? :lines
+        so[:lines] += b[:lines]
+      else
+        so[:lines] << as_line(b)
+      end
+
+      so
+    end
+
+    def self.merge a, b, options={}
+      options = @@default_options.merge options
+
+      bottom_left = [ [a[:x], b[:x]].min, [a[:y], b[:y]].min ]
+      top_right = [ [a[:x] + a[:width], b[:x] + b[:width]].max,
+                    [a[:y] + a[:height], b[:y] + b[:height]].max ]
+
+      so = a.merge(b).merge({
+        :x => bottom_left[0],
+        :y => bottom_left[1],
+        :width => top_right[0] - bottom_left[0],
+        :height => top_right[1] - bottom_left[1]
+      })
+
+      if options[:lines]
+        merge_lines a, b, so
+      else
+        so[:content] = (a[:content] + options[:separator] + b[:content])
+        so[:content] = so[:content].gsub /\s+/, " "
+      end
+      
+      if get_text_content(a).length > get_text_content(b).length
+        so[:font] = a[:font]
+        so[:line_height] = a[:line_height]
+      else
+        so[:font] = b[:font]
+        so[:line_height] = b[:line_height]
+      end
+
+      so
+    end
+
+    def self.line_count obj
+      line_count = 0
+      line_count += obj[:content].count("\n") + 1 if obj[:content]
+      line_count += obj[:lines].length if obj[:lines]
+      line_count
+    end
+
+    def self.get_dimensions obj
+      {
+        :x => obj[:x],
+        :y => obj[:y],
+        :width => obj[:width],
+        :height => obj[:height]
+      }
+    end
+
+    def self.as_line obj
+      get_dimensions(obj).merge({:content => obj[:content]})
+    end
+
+    def self.get_text_content obj
+      if obj[:lines]
+        obj[:lines].map do |line|
+          if line[:content] =~ /\-\Z/
+            line[:content][0..-2]
+          else
+            line[:content] + " "
+          end
+        end.join("").strip
+      elsif obj[:content]
+        obj[:content]
+      else
+        obj
+      end
+    end
+
+    # Collapse a list of objects into one. Will merge objects in the
+    # correct write order, specified by write_mode.
+    def self.collapse objs, options={}
+      options = @@default_options.merge options
+      
+      sorted = case write_mode
+               when :left_to_right
+                 objs.sort_by { |obj| -(obj[:y].floor * 100) + (obj[:x] / 100.0) }
+               end
+      
+      if sorted.count == 1
+        sorted.first.dup
+      else
+        o = sorted.delete_at(0).dup
+        while not sorted.count.zero?
+          merge o, sorted.delete_at(0)
+        end
+        o
+      end
+    end
+
+    def self.contains? a, b
+      a_x1 = a[:x]
+      a_x2 = a[:x] + a[:width]
+      a_y1 = a[:y]
+      a_y2 = a[:y] + a[:height]
+
+      b_x1 = b[:x]
+      b_x2 = b[:x] + b[:width]
+      b_y1 = b[:y]
+      b_y2 = b[:y] + b[:height]
+
+      b_x1 >= a_x1 && b_x2 <= a_x2 && b_y1 >= a_y1 && b_y2 <= a_y2 
+    end
+
+    def self.overlap? from, by, a, b
+      a_top = a[from] + a[by]
+      b_top = b[rom] + b[by]
+
+      (b_top <= a_top && b_top >= a[from]) || (b[from] >= a[from] && b[from] <= b_top)
+    end
+
+    def self.score items, ideals
+      types = {}
+      ideals.keys.each do |name|
+        types[name] = ideals[name].keys
+      end
+
+      types.each do |name, vars|
+        score_name = (name.to_s + "_score").to_sym
+        
+        vars.each do |var_name|
+
+          scores = []
+          items.each do |item|
+            diff = (item[var_name] - ideals[name][var_name][0]).abs
+            if diff.zero?
+              diff = Float::MIN
+            end
+            scores << 1.0 / diff
+          end
+
+          score_max = scores.max
+          weighted_scores = scores.map do |score|
+             (score / score_max) * ideals[name][var_name][1]
+          end
+
+          items.each_index do |idx|
+            items[idx][score_name] ||= 0.0
+            items[idx][score_name] += weighted_scores[idx]
+          end
+          
+        end
+      end
+    end
+    
+  end
+end

data/lib/view/abstract_view.rb

@@ -0,0 +1,32 @@
+module PdfExtract
+  class AbstractView
+
+    @@auto_colors = ["ff0000", "00ff00", "0000ff", "ffff00",
+                     "ff7f00", "ffc0cb", "800080", "f0e68c",
+                     "a52a2a"]
+
+    def initialize pdf, filename
+      @pdf = pdf
+      @filename = filename
+    end
+
+    # Return renderable objects - those whose spatials method was
+    # called explicitly.
+    def objects
+      @pdf.spatial_objects.reject { |type, _| not @pdf.explicit_call? type }
+    end
+
+    def auto_color
+      @next_auto_color = 0 if @next_auto_color.nil?
+      color = @@auto_colors[@next_auto_color]
+      @next_auto_color = @next_auto_color.next
+      color
+    end
+
+    def singular_name name
+      name = name.sub /ies$/, 'y'
+      name = name.sub /s$/, ''
+    end
+    
+  end
+end

data/lib/view/pdf_view.rb

@@ -0,0 +1,43 @@
+require 'prawn'
+require_relative 'abstract_view'
+
+module PdfExtract
+  class PdfView < AbstractView
+
+    def render options={}
+      Prawn::Document.new :template => @filename do |doc|
+        objects.each_pair do |type, objs|
+          last_page = 1
+          color = auto_color
+          doc.go_to_page last_page
+          doc.fill_color color
+          
+          objs.each do |obj|
+            unless obj[:page].nil?
+              if obj[:page] != last_page
+                last_page = obj[:page]
+                doc.go_to_page last_page
+                doc.fill_color color
+              end
+              
+              # XXX Works, but why?
+              pos = [obj[:x] - 36, obj[:y] + obj[:height] - 36]
+              width = obj[:width]
+              height = obj[:height]
+              
+              doc.transparent 0.2 do
+                doc.fill_rectangle pos, width, height
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def self.write render, filename
+      render.render_file filename
+    end
+    
+  end
+end
+

data/lib/view/png_view.rb

@@ -0,0 +1,30 @@
+require 'RMagick'
+
+require_relative 'abstract_view'
+
+module PdfExtract
+  class PngView < AbstractView
+
+    def render options={}
+      img = Magick::Image.new(800, 1000) { self.background_color = "white" }
+
+      objects.each_pair do |type, objs|
+        color = auto_color
+        objs.each do |obj|
+          gc = Magick::Draw.new
+          gc.fill = "\##{color}"
+          gc.rectangle(obj[:x], obj[:y], obj[:x] + obj[:width],
+                       obj[:y] + obj[:height])
+          gc.draw img
+        end
+      end
+        
+      img
+    end
+
+    def self.write render, filename
+      render.write filename
+    end
+    
+  end
+end

data/lib/view/xml_view.rb

@@ -0,0 +1,113 @@
+require 'nokogiri'
+
+require_relative 'abstract_view'
+require_relative '../language'
+
+module PdfExtract
+  class XmlView < AbstractView
+
+    @@ignored_attributes = [:content, :page, :page_width, :page_height]
+
+    @@numeric_attributes = [:x, :y, :width, :height, :line_height,
+                            :page_height, :page_width, :x_offset, :y_offset,
+                            :spacing, :letter_ratio, :cap_ratio, :year_ratio]
+
+    # Return renderable attributes
+    def get_xml_attributes obj
+      attribs = obj.reject { |k, _| @@ignored_attributes.include? k }
+      attribs = attribs.reject { |_, v| v.kind_of?(Hash) || v.kind_of?(Array) }
+      attribs.each_pair do |k, v|
+        if @@numeric_attributes.include?(k) || k.to_s =~ /.+_score/
+          attribs[k] = v.round(@render_options[:round])
+        end
+      end
+      attribs
+    end
+
+    def get_nested_objs obj
+      nested = obj.reject { |_, v| ! (v.kind_of?(Hash) || v.kind_of?(Array)) }
+      if @render_options[:lines]
+        nested
+      else
+        nested.reject { |k, _| k == :lines }
+      end
+    end
+
+    def render options={}
+      @render_options = {:lines => true, :round => 2, :outline => false}.merge(options)
+
+      pages = {}
+      page_params = {}
+      pageless = {}
+      
+      objects.each_pair do |type, objs|
+        objs.each do |obj|
+          if obj.key? :page
+            pages[obj[:page]] ||= {}
+            pages[obj[:page]][type] ||= []
+            
+            pages[obj[:page]][type] << obj
+            
+            page_params[obj[:page]] ||= {
+              :width => obj[:page_width],
+              :height => obj[:page_height],
+              :number => obj[:page]
+            }
+          else
+            pageless[type] ||= []
+            pageless[type] << obj
+          end
+        end
+      end
+
+      builder = Nokogiri::XML::Builder.new do |xml|
+        xml.pdf {
+          pageless.each_pair do |type, objs|
+            objs.each do |obj| write_obj_to_xml obj, type, xml end
+          end
+          
+          pages.each_pair do |page_number, obj_types|
+            xml.page(page_params[page_number]) {
+              obj_types.each_pair do |type, objs|
+                objs.each do |obj| write_obj_to_xml obj, type, xml end
+              end
+            }
+          end
+        }
+      end
+
+      builder.to_xml
+    end
+
+    def write_obj_to_xml obj, type, xml
+      xml.send singular_name(type.to_s), get_xml_attributes(obj) do
+
+        unless @render_options[:outline]
+          if not @render_options[:lines]
+            xml.text Language::transliterate(Spatial.get_text_content obj)
+          elsif obj.key?(:content)
+            xml.text Language::transliterate(obj[:content].to_s)
+          end
+        end
+
+        get_nested_objs(obj).each do |name, nested_obj|
+          element_name = singular_name name.to_s
+          if nested_obj.kind_of? Hash
+            write_obj_to_xml nested_obj, element_name, xml
+          elsif nested_obj.kind_of? Array
+            nested_obj.each do |item|
+              write_obj_to_xml item, element_name, xml
+            end
+          end
+        end
+      end
+    end
+
+    def self.write render, filename
+      File.open filename, "w" do |file|
+        file.write render
+      end
+    end
+    
+  end
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification 
+name: pdf-extract
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Karl Jonathan Ward
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-10-21 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: pdf-reader
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 0
+        - 0
+        - beta1
+        version: 1.0.0.beta1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        - 0
+        version: 1.5.0
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rmagick
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 13
+        - 1
+        version: 2.13.1
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: prawn
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 11
+        - 1
+        version: 0.11.1
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        - 4
+        version: 1.3.4
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: commander
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 4
+        - 0
+        - 4
+        version: 4.0.4
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  type: :runtime
+  version_requirements: *id007
+description: 
+email: 
+- kward@crossref.org
+executables: 
+- pdf-extract
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/assign.rb
+- bin/config.json
+- bin/fac_v19n11_s5.mask.pdf
+- bin/margins.mask.pdf
+- bin/one-column.mask.pdf
+- bin/pdf-extract
+- bin/s002040050107_Arch_Toxicol_1994_68_8.mask.pdf
+- bin/some3.mask.pdf
+- bin/some5.mask.pdf
+- bin/some6.mask.pdf
+- bin/train.rb
+- bin/two-column.mask.pdf
+- lib/analysis/columns.rb
+- lib/analysis/margins.rb
+- lib/analysis/sections.rb
+- lib/analysis/titles.rb
+- lib/analysis/zones.rb
+- lib/font_metrics.rb
+- lib/kmeans.rb
+- lib/language.rb
+- lib/model/characters.rb
+- lib/model/chunks.rb
+- lib/model/regions.rb
+- lib/multi_range.rb
+- lib/names.rb
+- lib/pdf-extract.rb
+- lib/pdf.rb
+- lib/references/references.rb
+- lib/references/resolve.rb
+- lib/references/resolved_references.rb
+- lib/spatial.rb
+- lib/view/abstract_view.rb
+- lib/view/pdf_view.rb
+- lib/view/png_view.rb
+- lib/view/xml_view.rb
+- data/familynames.db
+- data/stopwords.txt
+has_rdoc: true
+homepage: http://github.com/CrossRef/pdfextract
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 9
+      - 1
+      version: 1.9.1
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: PDF content extraction tool and library.
+test_files: []
+