pdf-extract 0.0.1

data/lib/model/characters.rb

@@ -0,0 +1,320 @@
+require 'matrix'
+
+require_relative '../font_metrics'
+
+module PdfExtract
+  module Characters
+
+    # TODO Implement writing mode and :FontMatrix.
+
+    def self.glyph_descent c, state
+      if state.last[:font_metrics].nil? || state.last[:font_metrics].descent.nil?
+        0
+      else
+        state.last[:font_metrics].descent / 1000.0
+      end
+    end
+
+    def self.glyph_ascent c, state
+      if state.last[:font_metrics].nil? || state.last[:font_metrics].ascent.nil?
+        0
+      else
+        state.last[:font_metrics].ascent / 1000.0
+      end
+    end
+    
+    def self.glyph_width c, state
+      # :Widths may be used to determine glyph width. This is the same as
+      # horizontal displacemnt.
+      glyph_displacement(c, state)[0]
+    end
+
+    def self.glyph_height c, state
+      # :Ascent and :Descent from the :FontDescriptor can be used to determine
+      # maximum glyph height.
+      glyph_ascent(c, state) - glyph_descent(c, state)
+    end
+
+    def self.glyph_displacement c, state
+      # For non-Type3 fonts, vertical displacement is the glyph width,
+      # horizontal displacement is always 0. Note glyph width is given
+      # in 1000ths of text units.
+      if state.last[:font_metrics].nil?
+        # XXX Why are some font resources not reported via resource_font?
+        # Bug in pdf-reader? Possibly because of :Font entry in graphics
+        # state set.
+        [ 0, 0 ]
+      else
+        [ state.last[:font_metrics].glyph_width(c) / 1000.0, 0 ]
+      end
+    end
+
+    def self.make_text_runs text, tj, state, render_state, page, page_number
+      # TODO Ignore chars outside the page :MediaBox.
+      # TODO Mul UserUnit if specified by page.
+      # TODO Include writing mode, so that runs can be joined either
+      #      virtically or horizontally in the join stage.
+      
+      objs = []
+      h_scale_mod = (state.last[:h_scale] / 100.0)
+      s = state.last
+      
+      disp_x, disp_y = [0, 0]
+      spacing = 0
+      tx = ((disp_x - (tj / 1000.0)) * s[:font_size] + spacing) * h_scale_mod
+      ty = (disp_y - (tj / 1000.0)) * s[:font_size] + spacing
+
+      # TODO Should use either tx or ty depending on writing mode.
+      render_state[:tm] = Matrix[ [1, 0, 0], [0, 1, 0], [tx, 0, 1] ] * render_state[:tm]
+
+      # tj applies only to the first char of the Tj op.
+      tj = 0
+      
+      text.each_char do |c|
+        trm = Matrix[ [s[:font_size] * h_scale_mod, 0, 0],
+                      [0, s[:font_size], 0],
+                      [0, s[:rise], 1] ]
+        trm = trm * render_state[:tm] * state.last[:ctm]
+
+        bl_pos = Matrix.rows( [ [0, glyph_descent(c, state), 1] ])
+        bl_pos = bl_pos * trm
+
+        width = glyph_width(c, state)
+        height = glyph_descent(c, state) + glyph_height(c, state)
+
+        tr_pos = Matrix.rows([ [width, height, 1] ])
+        tr_pos = tr_pos * trm
+
+        px = bl_pos.row(0)[0]
+        py = bl_pos.row(0)[1]
+        
+        objs << {
+          :x => px,
+          :y => py,
+          :width => tr_pos.row(0)[0] - px,
+          :height => tr_pos.row(0)[1] - py,
+          :line_height => tr_pos.row(0)[1] - py,
+          :content => state.last[:font].to_utf8(c),
+          :page => page_number,
+          :font => state.last[:font].basefont,
+          :page_width => page[:MediaBox][2] - page[:MediaBox][0],
+          :page_height => page[:MediaBox][3] - page[:MediaBox][1]
+        }
+        
+        disp_x, disp_y = glyph_displacement(c, state)
+        spacing = s[:char_spacing] if c != ' '
+        spacing = s[:word_spacing] if c == ' '
+        tx = ((disp_x - (tj / 1000.0)) * s[:font_size] + spacing) * h_scale_mod
+        ty = (disp_y - (tj / 1000.0)) * s[:font_size] + spacing
+
+        # TODO Should use either tx or ty depending on writing mode.
+        render_state[:tm] = Matrix[ [1, 0, 0], [0, 1, 0], [tx, 0, 1] ] * render_state[:tm]
+      end
+      
+      objs
+    end
+
+    def self.include_in pdf
+
+      pdf.spatials :characters do |parser|
+        state = []
+        page = nil
+        fonts = {}
+        font_metrics = {}
+        page_n = 0
+        render_state = {
+          :tm => Matrix.identity(3),
+          :tlm => Matrix.identity(3)
+        }
+
+        parser.for :resource_font do |data|
+          fonts[data[0]] = data[1]
+          font_metrics[data[0]] = FontMetrics.new data[1]
+          nil
+        end
+
+        parser.for :begin_page do |data|
+          page = data[0]
+          page_n = page_n.next
+          state << {
+            :h_scale => 100,
+            :char_spacing => 0,
+            :word_spacing => 0,
+            :leading => 0,
+            :rise => 0,
+            :font => nil,
+            :font_metrics => nil,
+            :font_size => 0,
+            :ctm => Matrix.identity(3)
+          }
+          nil
+        end
+
+        parser.for :end_page do |data|
+          state.pop
+          nil
+        end
+
+        parser.for :begin_text_object do |data|
+          render_state = {
+            :tm => Matrix.identity(3),
+            :tlm => Matrix.identity(3)
+          }
+          nil
+        end
+
+        # Graphics state operators.
+
+        parser.for :set_graphics_state_parameters do |data|
+          # TODO Handle gs graphics state dictionary set operation for
+          # :Font dictionary entries. Probably why font is sometimes nil.
+          # puts data
+          nil
+        end
+
+        parser.for :save_graphics_state do |data|
+          state.push state.last.dup
+          nil
+        end
+
+        parser.for :restore_graphics_state do |data|
+          state.pop
+          nil
+        end
+
+        parser.for :concatenate_matrix do |data|
+          a, b, c, d, e, f = data
+          ctm = state.last[:ctm]
+          state.last[:ctm] = Matrix[ [a, b, 0], [c, d, 0], [e, f, 1] ] * ctm
+          nil
+        end
+
+        # State change operators.
+
+        parser.for :set_text_leading do |data|
+          state.last[:leading] = data.first
+          nil
+        end
+
+        parser.for :set_text_rise do |data|
+          state.last[:rise] = data.first
+          nil
+        end
+
+        parser.for :set_character_spacing do |data|
+          state.last[:char_spacing] = data.first
+          nil
+        end
+
+        parser.for :set_word_spacing do |data|
+          state.last[:word_spacing] = data.first
+          nil
+        end
+
+        parser.for :set_horizontal_text_scaling do |data|
+          state.last[:h_scale] = data.first
+          nil
+        end
+
+        # Position change operators.
+
+        parser.for :move_text_position do |data|
+          render_state[:tm] = Matrix[
+            [1, 0, 0], [0, 1, 0], [data[0], data[1], 1]
+          ] * render_state[:tlm]
+          render_state[:tlm] = render_state[:tm]
+          nil
+        end
+
+        parser.for :move_text_position_and_set_leading do |data|
+          state.last[:leading] = -data[1]
+          render_state[:tm] = Matrix[
+            [1, 0, 0], [0, 1, 0], [data[0], data[1], 1]
+          ] * render_state[:tlm]
+          render_state[:tlm] = render_state[:tm]
+          nil
+        end
+
+        # Font change operators.
+
+        parser.for :set_text_font_and_size do |data|
+          state.last[:font] = fonts[data[0]]
+          state.last[:font_metrics] = font_metrics[data[0]]
+          state.last[:font_size] = data[1]
+          nil
+        end
+
+        # Text matrix change operators.
+
+        parser.for :set_text_matrix_and_text_line_matrix do |data|
+          # --     --
+          # | a b 0 |
+          # | c d 0 |
+          # | e f 1 |
+          # --     --
+          a, b, c, d, e, f = data
+          render_state[:tm] = Matrix[ [a, b, 0], [c, d, 0], [e, f, 1] ]
+          render_state[:tlm] = Matrix[ [a, b, 0], [c, d, 0], [e, f, 1] ]
+          nil
+        end
+
+        # New line operators.
+
+        parser.for :move_to_start_of_next_line do |data|
+          render_state[:tm] = Matrix[
+            [1, 0, 0], [0, 1, 0], [0, -state.last[:leading], 1]
+          ] * render_state[:tlm]
+          render_state[:tlm] = render_state[:tm]
+          nil
+        end
+
+        # Show text operators.
+
+        parser.for :set_spacing_next_line_show_text_raw do |data|
+          state.last[:word_spacing] = data[0]
+          state.last[:char_spacing] = data[1]
+          
+          render_state[:tm] = Matrix[
+            [1, 0, 0], [0, 1, 0], [0, -state.last[:leading], 1]
+          ] * render_state[:tlm]
+          render_state[:tlm] = render_state[:tm]
+
+          make_text_runs data[2], 0, state, render_state, page, page_n
+        end
+
+        parser.for :move_to_next_line_and_show_text_raw do |data|
+          render_state[:tm] = Matrix[
+            [1, 0, 0], [0, 1, 0], [0, -state.last[:leading], 1]
+          ] * render_state[:tlm]
+          render_state[:tlm] = render_state[:tm]
+          
+          make_text_runs data.first, 0, state, render_state, page, page_n
+        end
+
+        parser.for :show_text_raw do |data|
+          make_text_runs data.first, 0, state, render_state, page, page_n
+        end
+        
+        parser.for :show_text_with_positioning_raw do |data|
+          data = data.first
+          runs = []
+          tj = 0
+          
+          data.each do |item|
+            case item.class.to_s
+            when "Fixnum", "Float"
+              tj = item
+            when "String"
+              runs << make_text_runs(item, tj, state, render_state, page, page_n)
+              tj = 0
+            end
+          end
+          
+          runs.flatten
+        end
+        
+      end
+    end
+
+  end
+end

data/lib/model/chunks.rb

@@ -0,0 +1,103 @@
+require_relative '../spatial'
+
+module PdfExtract
+  module Chunks
+
+    # TODO Look for obj[:writing_mode] == :vertical or :horizontal
+
+    Settings.default :char_slop, 0.2
+    Settings.default :word_slop, 4.0
+    Settings.default :overlap_slop, 0.9
+
+    def self.include_in pdf
+      char_slop = 0.2
+      word_slop = 4.0
+      overlap_slop = 0.9
+      
+      pdf.spatials :chunks, :paged => true, :depends_on => [:characters] do |parser|
+        rows = {}
+
+        parser.before do
+          rows = {}
+        end
+        
+        parser.objects :characters do |chars|
+          y = chars[:y]
+          rows[y] = [] if rows[y].nil?
+
+          idx = rows[y].index { |obj| chars[:x] <= obj[:x] }
+          if idx.nil?
+            rows[y] << chars.dup
+          else
+            rows[y].insert idx, chars.dup
+          end
+        end
+
+        parser.after do
+          char_slop = pdf.settings[:char_slop]
+          word_slop = pdf.settings[:word_slop]
+          overlap_slop = pdf.settings[:overlap_slop]
+          
+          text_chunks = []
+
+          rows.each_pair do |y, row|
+            char_width = row.first[:width]
+
+            while row.length > 1
+              left = row.first
+              right = row[1]
+
+              if (left[:x] + left[:width] + (char_width * char_slop)) >= right[:x]
+                # join as adjacent chars
+                row[0] = Spatial.merge left, right
+                row.delete_at 1
+                char_width = right[:width] unless right[:content].strip =~ /[^A-Za-z0-9]/
+              elsif (left[:x] + left[:width] + (char_width * word_slop)) >= right[:x]
+                # join with a ' ' in the middle.
+                row[0] = Spatial.merge left, right, :separator => ' '
+                row.delete_at 1
+                char_width = right[:width] unless right[:content].strip =~ /[^A-Za-z0-9]/
+              else
+                # leave 'em be.
+                text_chunks << left
+                row.delete_at 0
+                char_width = row.first[:width]
+              end
+            end
+
+            text_chunks << row.first
+          end
+
+          # Merge chunks that have slightly different :y positions but which
+          # mostly overlap in the y dimension.
+
+          text_chunks.sort_by! { |obj| obj[:x] }
+          merged_text_chunks = []
+
+          while text_chunks.count > 1
+            left = text_chunks.first
+            right = text_chunks[1]
+
+            overlap = [left[:height], right[:height]].min - (left[:y] - right[:y]).abs
+            overlap = overlap / [left[:height], right[:height]].min
+
+            if overlap >= overlap_slop
+              # TODO follow char / word slop rules.
+              # join
+              text_chunks[0] = Spatial.merge left, right
+              text_chunks.delete_at 1
+            else
+              # no join
+              merged_text_chunks << text_chunks.first
+              text_chunks.delete_at 0
+            end
+          end
+
+          merged_text_chunks << text_chunks.first
+        end 
+      end
+    end
+
+  end
+end
+

data/lib/model/regions.rb

@@ -0,0 +1,112 @@
+require_relative '../spatial'
+
+module PdfExtract
+  module Regions
+
+    Settings.default :line_slop, 1.0
+
+    # TODO Handle :writing_mode once present in characters and text_chunks.
+
+    def self.incident l, r
+      lx1 = l[:x]
+      lx2 = l[:x] + l[:width]
+      rx1 = r[:x]
+      rx2 = r[:x] + r[:width]
+
+      lr = (lx1..lx2)
+      rr = (rx1..rx2)
+
+      lr.include? rx1 or lr.include? rx2 or rr.include? lx1 or rr.include? lx2
+    end
+
+    def self.append_line_offsets region
+      region[:lines] ||= []
+      region[:lines].each do |line|
+        line[:x_offset] = line[:x] - region[:x]
+        line[:y_offset] = line[:y] - region[:y]
+      end
+    end
+
+    def self.append_line_spacing region
+      region[:lines] ||= []
+      height_taken = 0
+      region[:lines].each do |line|
+        from_top = region[:height] - (line[:y_offset] + line[:height])
+        line[:spacing] = from_top - height_taken
+        height_taken = from_top + line[:height]
+      end
+    end
+    
+    def self.include_in pdf
+      pdf.spatials :regions, :paged => true, :depends_on => [:chunks] do |parser|
+        chunks = []
+        regions = []
+
+        parser.before do
+          chunks = []
+          regions = []
+        end
+        
+        parser.objects :chunks do |chunk|
+          y = chunk[:y].floor
+
+          idx = chunks.index { |obj| chunk[:y] <= obj[:y] }
+          if idx.nil?
+            chunks << chunk.dup
+          else
+            chunks.insert idx, chunk.dup
+          end
+        end
+
+        # TODO Rewrite to use Spatial::collapse so that text is in proper
+        # order.
+
+        parser.after do
+          # Convert chunks to have line content.
+          chunks.each do |chunk|
+            chunk[:lines] = [Spatial.as_line(chunk)]
+            chunk.delete :content
+          end
+          
+          compare_index = 1
+          while chunks.count > compare_index
+            b = chunks.first
+            t = chunks[compare_index]
+              
+            line_height = b[:line_height]
+            line_slop = [line_height, t[:height]].min * pdf.settings[:line_slop]
+            incident_y = (b[:y] + b[:height] + line_slop) >= t[:y]
+            
+            if incident_y && incident(t, b)
+              chunks[0] = Spatial.merge t, b, :lines => true
+              chunks.delete_at compare_index
+              compare_index = 1
+            elsif compare_index < chunks.count - 1
+              # Could be more chunks within range.
+              compare_index = compare_index.next
+            else
+              # Finished region.
+              regions << chunks.first
+              chunks.delete_at 0
+              compare_index = 1
+            end
+          end
+          
+          regions << chunks.first unless chunks.first.nil?
+
+          regions.each do |region|
+            append_line_offsets region
+            append_line_spacing region
+
+            region[:lines].map! do |line|
+              Spatial.drop_spatial line
+            end
+          end
+
+          regions.sort_by { |obj| -obj[:y] }
+        end
+      end
+    end
+    
+  end
+end

data/lib/multi_range.rb

@@ -0,0 +1,69 @@
+
+module PdfExtract
+  class MultiRange
+
+    attr_accessor :ranges
+
+    def initialize
+      @ranges = []
+    end
+
+    def append range
+      return if range.max.nil? || range.min.nil?
+
+      incident = @ranges.select do |r|
+        r.include?(range.min) || r.include?(range.max) ||
+          range.include?(r.min) || range.include?(r.max)
+      end
+      
+      incident << range
+
+      non_incident = @ranges - incident
+
+      non_incident << (incident.collect { |r| r.min }.min .. incident.collect { |r| r.max }.max)
+      @ranges = non_incident
+
+      @max_excluded = nil
+      @min_excluded = nil
+      @max = nil
+      @min = nil
+    end
+
+    def max_excluded
+      if @max_excluded.nil?
+        @max_excluded = @ranges.first.max if count == 1
+        @max_excluded = @ranges.sort_by { |r| -r.min }.first.min unless count == 1
+      end
+      @max_excluded
+    end
+
+    def min_excluded
+      if @min_excluded.nil?
+        @min_excluded = @ranges.first.min if count == 1
+        @min_excluded = @ranges.sort_by { |r| r.max }.first.max unless count == 1
+      end
+      @min_excluded
+    end
+
+    def max
+      @max ||= @ranges.sort_by { |r| -r.max }.first.max
+    end
+
+    def min
+      @min ||= @ranges.sort_by { |r| r.min }.first.min
+    end
+
+    def avg
+      @ranges.reduce(0) { |sum, r| sum += (r.max - r.min) } / @ranges.count.to_f
+    end
+
+    def covered
+      @ranges.reduce(0) { |total, r| total += (r.max - r.min) }
+    end
+
+    def count
+      @ranges.count
+    end
+
+  end
+end

data/lib/names.rb

@@ -0,0 +1,85 @@
+require "net/http"
+require "json"
+require "sqlite3"
+
+require_relative "pdf-extract"
+
+module PdfExtract::Names
+
+  class NamesDatabase
+    @@ambiguous_weighting = 0.1
+    @@unambiguous_weighting = 1.0
+
+    def self.path_to_data data_filename
+      File.join(File.dirname(File.expand_path(__FILE__)), "../data/" + data_filename)
+    end
+
+    @@db = SQLite3::Database.new(path_to_data("familynames.db"), {:readonly => true})
+    @@stop_words = File.open(path_to_data("stopwords.txt")).read.split(",")
+   
+    def self.detect_names content
+      words = content.split
+      sum = 0.0
+
+      words.each do |word|
+        word = word.downcase
+
+        if not @@stop_words.include? word && word.length > 1
+          query_word = word.capitalize.gsub(/-(.)/) { |s|
+            "-" + s[1].capitalize
+          }
+
+          @@db.execute("select * from names where name = ?", query_word) do |row|
+            if row[2] == 1
+              sum += @@ambiguous_weighting
+            else
+              sum += @@unambiguous_weighting
+            end
+          end
+        end
+        
+      end
+
+      if sum == 0
+        {:name_frequency => 0}
+      else
+        {:name_frequency => (sum / words.length.to_f)}
+      end
+    end
+  end
+      
+  class NamesService
+    def self.detect_names content
+      data = {:name_frequency => 0.0}
+      begin
+        response = Net::HTTP.start "names.crrd.dyndns.org" do |http|
+          http.post "/detect", content
+        end
+      
+        if response.code == "200"
+          data = JSON.parse response.body
+        end
+      rescue
+      end
+      data
+    end
+  end
+
+  class NoDetection
+    def self.detect_names content
+      {:name_frequency => 0.0}
+    end
+  end
+
+  @@detector = NamesDatabase
+
+  def self.detector= detector_class
+    @@detector = detector_class
+  end
+
+  def self.detect_names content
+    @@detector.detect_names content
+  end
+  
+end
+