pdf-extract 0.0.1

data/lib/pdf-extract.rb

@@ -0,0 +1,77 @@
+require_relative 'pdf'
+require_relative 'model/characters'
+require_relative 'model/chunks'
+require_relative 'model/regions'
+require_relative 'analysis/titles'
+require_relative 'analysis/margins'
+require_relative 'analysis/zones'
+require_relative 'analysis/columns'
+require_relative 'analysis/sections'
+require_relative 'references/references'
+require_relative 'references/resolved_references'
+require_relative 'view/png_view'
+require_relative 'view/pdf_view'
+require_relative 'view/xml_view'
+
+module PdfExtract
+
+  @views = {}
+
+  @parsers = []
+
+  def self.add_view name, view_class
+    @views[name] = view_class
+  end
+
+  def self.add_parser parser_class
+    @parsers << parser_class
+  end
+  
+  def self.parse filename, &block
+    pdf = Pdf.new
+
+    @parsers.each do |p|
+      p.include_in pdf
+    end
+    
+    yield pdf
+    
+    pdf.spatial_calls.each do |spatial_call|
+      name = spatial_call[:name]
+      receiver = Receiver.new pdf
+      pdf.spatial_builders[name].call receiver
+      receiver.invoke_calls filename, pdf.spatial_options[name]
+    end
+    
+    pdf
+  end
+  
+  def self.view_class short_name
+    @views[short_name]
+  end
+
+  def self.view filename, options = {}, &block
+    pdf = parse filename, &block
+    view_class(options[:as]).new(pdf, filename).render options
+  end
+
+  def self.init
+    add_parser Characters
+    add_parser Chunks
+    add_parser Regions
+    add_parser Titles
+    add_parser Margins
+    add_parser Zones
+    add_parser Columns
+    add_parser Sections
+    add_parser References
+    add_parser ResolvedReferences
+
+    add_view :pdf, PdfView
+    add_view :png, PngView
+    add_view :xml, XmlView
+  end
+
+  init
+
+end

data/lib/pdf.rb

@@ -0,0 +1,255 @@
+require 'pdf-reader'
+
+module PdfExtract
+
+  class Settings
+    
+    @@defaults = {}
+    
+    def self.default key, default_value
+      @@defaults[key] = default_value
+    end
+    
+    def initialize
+      @settings = {}
+      @agents = {}
+    end
+  
+    def [] key
+      @settings[key] || @@defaults[key] ||
+        raise("Attempt to use undeclared setting \"#{key}\"")
+    end
+    
+    def set key, value, agent=""
+      if @@defaults[key]
+        @settings[key] = value.to_f
+        @agents[key] = agent
+      else
+        raise "Attempt to set an undefined setting \"#{key}\""
+      end
+    end
+
+    def unmodified
+      @@defaults.reject { |k, v| @settings[k] }
+    end
+
+    def modified
+      @settings
+    end
+
+    def agent key
+      @agents[key]
+    end
+    
+  end
+  
+  class Receiver
+
+    def initialize pdf
+      @pdf = pdf
+      @listeners = {}
+      @object_listeners = {}
+    end
+
+    def for callback_name, &block
+      @listeners[callback_name] = {:type => @pdf.operating_type, :fn => block}
+    end
+
+    def objects type_name, &block
+      @object_listeners[type_name] ||= []
+      @object_listeners[type_name] << block
+    end
+
+    def before &block
+      @before = block
+    end
+
+    def after &block
+      @after = {:type => @pdf.operating_type, :fn => block}
+    end
+
+    def expand_listeners_to_callback_methods
+      # TODO merge on callback_name
+      @listeners.each_pair do |callback_name, callback_handler|
+        p = proc do |*args|
+          spatial_objects = callback_handler[:fn].call args
+          self.add_spatial_objects callback_handler[:type], spatial_objects
+        end
+        
+        self.class.send :define_method, callback_name, p
+      end
+    end
+
+    def call_object_listeners spatial_objects
+      @object_listeners.each_pair do |type, listeners|
+        listeners.each do |listener|
+            spatial_objects[type].each { |obj| listener.call obj }
+        end
+      end
+    end
+
+    def call_after
+      self.add_spatial_objects @after[:type], @after[:fn].call unless @after.nil?
+    end
+
+    def call_before
+      @before.call unless @before.nil?
+    end
+
+    def for_calls?
+      @listeners.size > 0
+    end
+
+    def object_calls?
+      @object_listeners.size > 0
+    end
+
+    def add_spatial_objects default_type, objs
+      if objs.class != Array
+        objs = [objs] unless objs.nil?
+        objs = [] if objs.nil?
+      end
+
+      objs.each do |obj|
+        type = obj.delete(:group) || default_type
+        @pdf.spatial_objects[type] ||= []
+        @pdf.spatial_objects[type] << obj
+      end
+    end
+
+    def invoke_calls filename, spatial_options
+      if spatial_options[:paged]
+          
+        paged_objs = {}
+        @object_listeners.each_pair do |type, _|
+          @pdf.paged_objects(type).each_pair do |page, objs|
+              paged_objs[page] ||= {}
+            paged_objs[page][type] = objs
+          end
+        end
+        
+        paged_objs.each_pair do |page, objs|
+          self.call_before
+
+          if self.object_calls?
+            @object_listeners.each_pair do |type, listeners|
+              listeners.each do |listener|
+                if objs[type].nil?
+                  raise "#{@pdf.operating_type} is missing a dependency on #{type}"
+                end
+                objs[type].each { |obj| listener.call obj }
+              end
+            end
+          end
+          
+          self.call_after
+        end
+        
+      else
+
+        self.call_before
+        if self.object_calls?
+          self.call_object_listeners @pdf.spatial_objects
+        end
+        self.call_after
+
+      end
+      
+      if self.for_calls?
+        self.expand_listeners_to_callback_methods
+        PDF::Reader.file filename, self, :raw_text => true
+      end
+    end
+
+  end
+
+  class Pdf
+    
+    attr_accessor :operating_type, :spatial_calls, :spatial_builders, :spatial_objects
+    attr_accessor :spatial_options, :settings
+    
+    def method_missing name, *args
+      raise "No such spatial type #{name}"
+    end
+
+    def spatials name, options = {}, &block
+      add_spatials_method name, options, &block
+    end
+
+    def initialize
+      @spatial_builders = {}
+      @spatial_calls = []
+      @spatial_objects = {}
+      @spatial_options = {}
+      @settings = Settings.new
+    end
+
+    def explicit_call? name
+      @spatial_calls.count { |obj| obj[:name] == name and obj[:explicit] } > 0
+    end
+
+    def paged_objects type
+      paged_objs = {}
+      
+      if @spatial_objects[type]
+        @spatial_objects[type].each do |obj|
+          paged_objs[obj[:page]] ||= []
+          paged_objs[obj[:page]] << obj
+        end
+      end
+
+      paged_objs
+    end
+
+    def [](type)
+      @spatial_objects[type]
+    end
+
+    def set setting, value, agent=""
+      @settings.set setting, value, agent
+    end
+
+    private
+
+    def append_deps deps_list
+      # TODO if explicit is true, overwrite non-explicit deps.
+      deps_list.each do |dep|
+        append_deps @spatial_options[dep].fetch(:depends_on, [])
+        if @spatial_calls.count { |obj| obj[:name] == dep }.zero?
+          @spatial_calls << {
+            :name => dep,
+            :explicit => false
+          }
+        end
+      end
+    end
+    
+    def add_spatials_method name, options={}, &block
+      options = {:depends_on => [], :defined_by => []}.merge options
+      
+      @spatial_objects[name] = []
+      @spatial_builders[name] = proc { |receiver|
+        @operating_type = name
+        block.call receiver unless block.nil?
+      }
+      @spatial_options[name] = options
+
+      p = Proc.new do
+        append_deps options[:depends_on]
+
+        @spatial_calls << {
+          :name => name,
+          :explicit => true
+        }
+        
+        @spatial_objects[name].each do |o|
+          yield o
+        end
+      end
+
+      self.class.send :define_method, name, p
+    end
+  end
+  
+end
+

data/lib/references/references.rb

@@ -0,0 +1,184 @@
+require_relative "../spatial"
+
+module PdfExtract
+  module References
+ 
+    Settings.default :min_score, 6.4
+    Settings.default :min_sequence_count, 3
+    Settings.default :max_reference_order, 1000
+
+    def self.partition_by ary, &block
+      matching = []
+      parts = []
+      ary.each do |item|
+        if yield(item)
+          parts << matching
+          matching = []
+        end
+        matching << item
+      end
+      parts.reject { |p| p.empty? }
+    end
+
+    def self.frequencies lines, delimit_key
+      fs = {}
+      lines.each do |line|
+        val = line[delimit_key].floor
+        fs[val] ||= 0
+        fs[val] = fs[val].next
+      end
+
+      ary = []
+      fs.each_pair do |key, val|
+        ary << {:value => key, :count => val}
+      end
+
+      ary.sort_by { |item| item[:count] }.reverse
+    end
+
+    def self.select_delimiter lines, delimit_key
+      frequencies(lines, delimit_key)[1][:value]
+    end
+
+    def self.split_by_margin lines
+      delimiting_x_offset = select_delimiter lines, :x_offset
+      lines = lines.drop_while { |l| l[:x_offset].floor != delimiting_x_offset }
+      parts = partition_by(lines) { |line| line[:x_offset].floor == delimiting_x_offset }
+      parts.map { |part| {:content => part.map { |line| line[:content] }.join(" ")} }
+    end
+
+    def self.split_by_line_spacing lines
+      delimiting_spacing = select_delimiter lines, :spacing
+      lines = lines.drop_while { |l| l[:spacing].floor != delimiting_spacing }
+      parts = partition_by(lines) { |line| line[:spacing].floor == delimiting_spacing }
+      parts.map { |part| {:content => part.map { |line| line[:content] }.join(" ")} }
+    end
+
+    def self.split_by_delimiter pdf, s
+      # Find sequential numbers and use them as partition points.
+
+      # Determine the charcaters that are most likely part of numeric
+      # delimiters.
+      
+      after = {}
+      before = {}
+      last_n = -1
+      
+      s.scan /[^\d]?\d+[^\d]/ do |m|
+        n = m[/\d+/].to_i
+        if n < pdf.settings[:max_reference_order]
+          if last_n == -1
+            before[m[0]] ||= 0
+            before[m[0]] = before[m[0]].next
+            after[m[-1]] ||= 0
+            after[m[-1]] = after[m[-1]].next
+            last_n = n
+          elsif n == last_n.next
+            before[m[0]] ||= 0
+            before[m[0]] = before[m[0]].next
+            after[m[-1]] ||= 0
+            after[m[-1]] = after[m[-1]].next
+            last_n = last_n.next
+          end
+        end
+      end
+
+      b_s = "" if before.length.zero?
+      b_s = "\\" + before.max_by { |_, v| v }[0] unless before.length.zero?
+      a_s = "" if after.length.zero?
+      a_s = "\\" + after.max_by { |_, v| v }[0] unless after.length.zero?
+
+      # TODO Turn into settings. Needs typed settings
+      if ["", "\\[", "\\ "].include?(b_s) && ["", "\\.", "\\]", "\\ "].include?(a_s)
+
+        # Split by the delimiters and record separate refs.
+        
+        last_n = -1
+        current_ref = ""
+        refs = []
+        parts = s.partition(Regexp.new "#{b_s}?\\d+#{a_s}")
+        
+        while not parts[1].length.zero?
+          n = parts[1][/\d+/].to_i          
+          if n < pdf.settings[:max_reference_order] && last_n == -1
+            last_n = n
+          elsif n == last_n.next
+            current_ref += parts[0]
+            refs << {
+              :content => current_ref.strip,
+              :order => last_n
+            }
+            current_ref = ""
+            last_n = last_n.next
+          else
+            current_ref += parts[0] + parts[1]
+          end
+
+          parts = parts[2].partition(Regexp.new "#{b_s}?\\d+#{a_s}")
+        end
+        
+        refs << {
+          :content => (current_ref + parts[0]).strip,
+          :order => last_n
+        }
+        
+        refs
+
+      else
+        []
+      end
+    end
+
+    def self.multi_margin? lines
+      lines.uniq { |line| line[:x_offset].floor }.count > 1
+    end
+
+    def self.multi_spacing? lines
+      lines.uniq { |line| line[:spacing].floor }.count > 1
+    end
+
+    def self.numeric_sequence? pdf, content
+      last_n = -1
+      seq_count = 0
+      content.scan /\d+/ do |m|
+         # Avoid misinterpreting years as sequence
+        if m.to_i < pdf.settings[:max_reference_order]
+          if last_n == -1
+            last_n = m.to_i
+          elsif last_n.next == m.to_i
+            last_n = last_n.next
+            seq_count = seq_count.next
+          end
+        end
+      end
+
+      seq_count >= pdf.settings[:min_sequence_count]
+    end
+    
+    def self.include_in pdf
+      pdf.spatials :references, :depends_on => [:sections] do |parser|
+
+        refs = []
+
+        parser.objects :sections do |section|
+          # TODO Take top x%, fix Infinity coming back from score.
+          if section[:reference_score] >= pdf.settings[:min_score]
+            if numeric_sequence? pdf, Spatial.get_text_content(section)
+              refs += split_by_delimiter pdf, Spatial.get_text_content(section)
+            elsif multi_margin? section[:lines]
+              refs += split_by_margin section[:lines]
+            elsif multi_spacing? section[:lines]
+              refs += split_by_line_spacing section[:lines]
+            end
+          end
+        end
+
+        parser.after do
+          refs
+        end
+
+      end
+    end
+
+  end
+end

data/lib/references/resolve.rb

@@ -0,0 +1,113 @@
+require 'cgi'
+require 'nokogiri'
+require 'open-uri'
+require 'net/http'
+
+module PdfExtract::Resolve
+
+  class Sigg
+
+    def self.find ref
+      url = "http://api.labs.crossref.org/search?q=#{CGI.escape(ref)}"
+      resolved = {}
+      begin
+        doc = Nokogiri::HTML(open url)
+      
+        result = doc.at_css "div.result"
+        unless result.nil?
+          score = result.at_css("span.cr_score").content.to_s
+          if score.to_i >= 90
+            doi = result.at_css "span.doi" 
+            resolved[:doi] = doi.content.sub "http://dx.doi.org/", ""
+          end
+        end
+      rescue
+      end
+      resolved
+    end
+    
+  end
+  
+  class FreeCite
+    
+    def self.find ref
+      Net::HTTP.start "freecite.library.brown.edu" do |http|
+        r = http.post "/citations/create", "citation=#{ref}",
+                      "Accept" => "text/xml"
+        doc = Nokogiri::XML r.body
+        
+        {
+          :title => doc.at_xpath("//title").content,
+          :journal => doc.at_xpath("//journal").content,
+          :pages => doc.at_xpath("//pages").content,
+          :year => doc.at_xpath("//year").content
+        }
+      end
+    end
+    
+  end
+  
+  class SimpleTextQuery
+
+    @@cookie = nil
+    
+    def self.find ref
+      create_session
+
+      post = Net::HTTP::Post.new "/SimpleTextQuery"
+      post.add_field "Cookie", @@cookie
+      post.add_field "Referer", "http://www.crossref.org/SimpleTextQuery"
+      post.set_form_data({
+        "command" => "Submit",
+        "freetext" => ref,
+        #"emailField" => "kward@crossref.org",
+        "doiField" => "",
+        #"username" => "",
+        #"password" => ""
+      })
+      response = Net::HTTP.start "www.crossref.org" do |http|
+        http.request post
+      end
+      
+      doc = Nokogiri::HTML response.body
+      doi = doc.at_css "td.resultB > a"
+      
+      if doi.nil?
+        {}
+      else
+        {:doi => doi.content.sub("doi:", "")}
+      end
+    end
+
+    def self.create_session
+      if @@cookie.nil?
+        Net::HTTP.start "www.crossref.org" do |http|
+          response = http.get "/SimpleTextQuery"
+          @@cookie = response["Set-Cookie"]
+        end
+      end
+    end
+    
+  end
+  
+  @@resolvers = [Sigg]
+  
+  def self.resolvers= resolver
+    @@resolvers = resolver
+  end
+
+  def self.add_resolver resolver
+    unless @@resolvers.contains? resolver
+      @@resolvers << resolver
+    end
+  end
+
+  def self.find ref
+    ref = ref.dup
+    @@resolvers.each do |resolver|
+      ref.merge! resolver.find(ref[:content])
+    end
+    ref
+  end
+  
+end

data/lib/references/resolved_references.rb

@@ -0,0 +1,37 @@
+require_relative 'resolve'
+
+module PdfExtract
+  module ResolvedReferences
+
+    def self.include_in pdf
+      pdf.spatials :resolved_references, :depends_on => [:references] do |parser|
+
+        resolved_refs = []
+        
+        parser.objects :references do |ref|
+          resolved_refs << ref.merge(Resolve.find(ref))
+        end
+
+        parser.after do
+          resolved_refs
+        end
+
+      end
+    end
+
+    def self.reverse_resolve ref
+      
+      url = "http://api.labs.crossref.org/search?q=#{CGI.escape(ref)}"
+      doc = Nokogiri::HTML(open url)
+
+      result = doc.at_css "div.result"
+      score = result.at_css("span.cr_score").content.to_s
+      if score.to_i >= 90
+        result.at_css("span.doi").content.sub("http://dx.doi.org/", "")
+      else
+        ""
+      end
+    end
+
+  end
+end