rpdf2txt 0.8.2

data/lib/rpdf2txt/text.rb

@@ -0,0 +1,182 @@
+#!/usr/bin/env ruby
+#
+#	Rpdf2txt -- PDF to Text Parser
+#	Copyright (C) 2003 Andreas Schrafl, Hannes Wyss
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Z�rich, Switzerland
+#	hwyss@ywesee.com,	aschrafl@ywesee.com
+#
+# Text -- Rpdf2txt -- 28.11.2002 -- aschrafl@ywesee.com
+
+require 'rpdf2txt/text_state'
+require 'rpdf2txt/textparser'
+require 'rpdf2txt/object'
+require 'iconv'
+
+module Rpdf2txt
+	class Text
+		attr_writer :current_page
+		attr_reader :text_state, :transformation_matrix
+		def initialize(src, target_encoding='utf8', 
+                   tm=Matrix[[1,0,0],[0,-1,0],[0,0,1]])
+			@src = src.gsub(/[\r\n]+/n, "\n")
+			@text_state = TextState.new(target_encoding)
+			@transformation_matrix = tm
+			@text_state.transformation_matrix = tm
+		end
+		## FIXME: generic_symbol_font is a workaround. Implement a way to 
+		##        pass unicode-snippets (or rework everything to unicode)
+		def generic_symbol_font(font)
+			if(font.nil?)
+				Font.new('<< /BaseFont /Symbol')
+			elsif(/symbol/in.match(font.basefont_name))
+				font
+			else
+				genfont = font.dup
+				genfont.attributes[:basefont] = 'Symbol'
+				genfont
+			end
+		end
+		def get_font(font_name)
+			return nil unless @current_page
+			@current_page.font(font_name.to_s.downcase.intern) 
+		end
+		def mapped_ascii(ascii)
+			if(@current_font)
+        if((cmap = @current_font.cmap) && (map = cmap.map) \
+           && (unicode_bytes = map[ascii]) \
+           && (ascii = SymbolMap::SYMBOL_ENTITIES[unicode_bytes]))
+          ascii.chr
+        elsif((map = @current_font.to_unicode) \
+            && (utf8 = map.to_utf8(ascii)))
+          @current_font.attributes[:encoding] = '/UTF8'
+          #@text_state.set_font(@current_font)
+          [utf8].pack('U')
+        end
+      end
+		end
+		def scan
+			@snippets = []
+			ast = Rpdf2txt.text_parser.parse(@src)
+			scan_tree(ast)
+			@snippets
+    rescue Exception
+      puts @src
+      raise
+		end
+		def scan_tree(ast)
+			ast.values.each { |node|
+				if(node.name == 'Array') \
+					&& (node.values.first.children_names.first == 'kerning')
+					## If the case [ 34 (foo) ] crops up, the first operation 
+					## executed on @text_state is advance_x. This results in 
+					## the width of the last text-snipped being calculated twice.
+					## This here is a workaround that resets the snippet to an 
+					## empty string if we are encountering a [ ??? ] construct
+					## (an array).
+					## TODO: find a more general solution
+					@text_state.set_txt('')
+				end
+				node.children_names.each { |child_name|
+					case child_name
+          when 'alpha'
+            @text_state.tmalpha = node.alpha.value.to_f
+          when 'beta'
+            @text_state.tmbeta = -node.beta.value.to_f
+            skew = node.beta.value.to_f > 0.1
+            if(@current_font && @current_font.skewed != skew)
+              @current_font = @current_font.dup
+              @current_font.skewed = skew
+						  @text_state.set_font(@current_font)
+            end
+					when 'xscale'
+						@text_state.set_xscale(node.xscale.value)
+					when 'yscale'
+						@text_state.set_yscale(node.yscale.value)
+					when 'charspace'
+						@text_state.set_char_spacing(node.charspace.value)
+					when 'kerning'
+						@text_state.advance_x(node.kerning.value.to_f)
+					when 'tdleadx'
+						@text_state.update_x(node.tdleadx.value.to_f)
+					when 'tdleady'
+						lead = node.tdleady.value.to_f
+						@text_state.set_lead(lead)
+						@text_state.update_y(lead)
+					when 'xpos'
+						@text_state.update_x(node.xpos.value.to_f)
+					when 'ypos'
+						@text_state.update_y(node.ypos.value.to_f)
+					when 'fontname'
+						@current_font = get_font(node.fontname.value)
+						@text_state.set_font(@current_font)
+						@text_state.set_font_size(node.fontsize.value)
+					when 'tmx'
+						@text_state.set_x(node.tmx.value.to_f)
+					when 'tmy'
+						@text_state.set_y(node.tmy.value)
+          when 'render'
+            val = node.render.value
+            if(@current_font && @current_font.rendering_mode != val)
+              @current_font = @current_font.dup
+              @current_font.rendering_mode = val
+						  @text_state.set_font(@current_font)
+            end
+					when 'wordspace'
+						@text_state.set_word_spacing(node.wordspace.value)
+					when 'values'
+						scan_tree(node)
+					when 'snippet'
+						snip(node.snippet.value)
+					when 'aposnippet'
+						@text_state.step
+						snip(node.aposnippet.value)
+					when 'linebreak'
+						@text_state.step
+					when 'textrise'
+						#add functionality for textrise p 387 pdf manual
+					when 'hexsnippet'
+						hex_bytes = node.hexsnippet.value
+            char = ''
+            hex_bytes.scan(/.{2,4}/n) { |pair|
+              dec_byte = pair.hex
+              char << (mapped_ascii(dec_byte) || '?')
+            }
+						_snip(char)
+					end
+				}
+			}
+		end
+		def snip(snippet)
+			snippet_text = snippet[1..-2].gsub(/\\[nrt]/n, " ")
+      snippet_text.gsub!(/\\([()])/n, '\1')
+			snippet_text.gsub!(/./n) { |char|
+        self.mapped_ascii(char[0]) || char
+			}
+			_snip(snippet_text)
+		end
+		def _snip(snippet_text)
+			@text_state.set_txt(snippet_text)
+      @text_state.update!(@current_page ? @current_page.attributes[:rotate] : 0)
+			@snippets.push(@text_state.dup).last
+		end
+		def text_state=(text_state)
+			text_state.transformation_matrix = @transformation_matrix
+			@text_state = text_state
+		end
+	end
+end

data/lib/rpdf2txt/text_state.rb

@@ -0,0 +1,434 @@
+#!/usr/bin/env ruby
+#
+#	Rpdf2txt -- PDF to Text Parser
+#	Copyright (C) 2003 Andreas Schrafl, Hannes Wyss
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Z�rich, Switzerland
+#	hwyss@ywesee.com,	aschrafl@ywesee.com
+#
+# TextState -- Rpdf2txt -- 29.11.2002 -- asschrafl@ywesee.com
+
+module Rpdf2txt
+  class PositionedElement
+		USER_SPACE = 1000.0
+    attr_accessor :media_box, :tmalpha, :tmbeta
+    def fire_callbacks(previous_positioned_element, callback_handler)
+    end
+    def set_x(x)
+      @tmx = @dtmx = 0
+      @tmxoffset = x.to_f
+    end
+    def set_xscale(xscale)
+      @tmxscale = (xscale.to_f * USER_SPACE).round.to_f / USER_SPACE
+    end
+    def set_y(y)
+      @tmy = 0
+      @tmyoffset = y.to_f
+    end
+    def set_yscale(yscale)
+      @tmyscale = (yscale.to_f * USER_SPACE).round.to_f / USER_SPACE
+    end
+    def transformation_matrix=(tm)
+      ### This shouldn't happen, but we do have an example of
+      ### it happening in 
+      ### /Producer (Hyf PDF Output Library 2.2.3 \(Windows\))
+      ### /Producer (Mac OS X 10.4.6 Quartz PDFContext)
+      @cmxscale = (tm[0,0] * USER_SPACE).round.to_f / USER_SPACE
+      @cmalpha = tm[0,1]
+      @cmbeta = tm[1,0]
+      @cmyscale = (tm[1,1] * USER_SPACE).round.to_f / USER_SPACE
+      @cmxoffset = tm[2,0]
+      @cmyoffset = tm[2,1]
+    end
+    def whitespace_overlap?(previous)
+      false
+    end
+    def update!(rotation=0)
+      orientation = (rotation.to_f.round / 90) % 2
+      x, y, x2, y2, bx, by = nil
+      if orientation == 1
+        x = @tmxoffset + @tmy * @tmalpha
+        y = @tmyoffset + (@tmx + @dtmx) * @tmbeta
+        x2 = bx = x + @font_size * @tmalpha
+        y2 = y + @w * @tmbeta
+        by = y + @boxwidth * @tmbeta
+        @x = y + @cmxoffset
+        @y = x + @cmyoffset
+        @x2 = y2 + @cmxoffset
+        @y2 = x2 + @cmyoffset
+        @right_edge = by + @cmxoffset
+      else
+        x = @tmxoffset + (@tmx + @dtmx) * @tmxscale
+        y = @tmyoffset - @tmy * @tmyscale
+        x2 = x + @w * @tmxscale
+        y2 = by = y - @font_size * @tmyscale
+        bx = x + @boxwidth * @tmxscale
+        @x = x + @cmxoffset
+        @y = y + @cmyoffset
+        @x2 = x2 + @cmxoffset
+        @y2 = y2 + @cmyoffset
+        @right_edge = bx + @cmxoffset
+      end
+    end
+    def <=> (other)
+      if(same_line(other))
+        @x <=> other.x
+      elsif(other.is_a?(self.class))
+        # @cmyscale may be negative, reversing the sort-order
+        (@y <=> other.y) \
+          * (@cmyscale == 0 ? 1 : @cmyscale)
+      else
+        @y <=> other.y
+      end
+    end
+  end
+	class TextState < PositionedElement
+    UTF = /utf/in
+		attr_accessor :font, :txt
+		include Comparable
+		attr_reader :y, :x, :x2, :y2, :w, :boxwidth, :xscale, :font_size, :yscale,
+                :right_edge
+		def initialize(target_encoding='utf8')
+      @boxwidth = 0
+			@x = @tmx = @dtmx = @tmxoffset = @cmxoffset = 0.0
+			@y = @tmy = @tmyoffset = @cmyoffset = 0.0
+			@w = 0.0
+      @tmalpha = @cmalpha = 0.0
+      @tmbeta = @cmbeta = 0.0
+			@tmxscale = @cmxscale = 1.0
+			@tmyscale = @cmyscale = 1.0
+			@lead = nil
+			@font = nil
+			@font_size = 1
+			@char_spacing = 0
+			@word_spacing = 0
+			@target_encoding = target_encoding + '//TRANSLIT//IGNORE'
+			self.transformation_matrix = Matrix[[1,0,0],[0,1,0],[0,0,1]]
+		end
+		def advance_x(kerning = 0)
+			@dtmx += @w - kerning/USER_SPACE
+		end
+		def char_width(char)
+			if(char.is_a? String)
+				char = char[0]
+			end
+			w = 0.0
+			if(@font && (width = @font.width(char)))
+				w = width 
+			elsif(@font && (avg = @font.attributes[:avgwidth]))
+				w = avg
+			end
+      w = 300.0 if w == 0
+			w += @char_spacing
+			if(char==32)
+				w += @word_spacing
+			end
+			w * @font_size / USER_SPACE
+		end
+    def whitespace_overlap?(previous)
+      previous && empty? && same_line(previous) \
+        && previous.x2 >= (@x + (@x2 - @x) / 2)
+    end
+    def send_content(previous, callback_handler)
+			if(previous)
+				if(previous.font != @font)
+					callback_handler.new_font(@font)
+				end
+				if(previous.font_height != self.font_height)
+					callback_handler.new_fontsize(self.font_height)
+				end
+			else
+				callback_handler.new_font(@font)
+      end
+      callback_handler.send_flowing_data(@txt)
+    end
+		def space_width
+			w = 300.0
+			if(@font && (width = @font.width(32)))
+				w = width
+			elsif(@font && (avg = @font.attributes[:avgwidth]))
+				w = avg
+			end
+			w += @char_spacing
+			w * @font_size / USER_SPACE
+		end
+		def fire_early_callbacks(previous, callback_handler)
+			if(previous)
+				if(!same_line(previous))
+          callback_handler.send_line_break
+				elsif(!same_word(previous))
+					if(spaces = previous.count_spaces(@x - previous.x2))
+						callback_handler.send_flowing_data(' '*spaces.abs)
+					end
+				end
+				if(new_paragraph(previous))
+					callback_handler.send_paragraph
+				end
+			end
+		end
+    def count_lines(displacement)
+      (displacement / lead).abs.ceil
+    rescue
+      1
+    end
+    def count_spaces(displacement)
+      x = space_width * @tmxscale + @font_size * @tmalpha
+      y = @font_size * @tmyscale + space_width * @tmbeta
+      width = x * @cmxscale + y * @cmalpha
+      if(width.nonzero? && displacement > width)
+        (displacement / width).round
+      end
+    rescue ZeroDivisionError
+      warn "Ignoring Division by Zero: #{displacement.inspect}/#{width.inspect}"
+    end
+    def empty?
+      @txt.nil? || @txt.strip.empty?
+    end
+		def lead
+			@lead || -font_height * 1.2
+		end
+		def font_height
+			@font_size
+		end
+		def new_paragraph(last_text_state)
+			return false if(last_text_state.font_size.nil?)
+			#1.5 is an approximate value
+			spacing = last_text_state.font_height * 1.5
+			last_y = last_text_state.y
+			((last_y - @y).abs > spacing.abs)
+		end
+		def recode_txt(txt)
+      enc = @font.encoding
+      if(enc.is_a?(Encoding))
+        # it would certainly be nice to do without all this iconving,
+        # but since CMaps always contain utf8, and using utf16 in 
+        # Symbol.from_* is so much more practical than dealing with 
+        # variable-length utf8 encoding for the characters in the
+        # Symbol font, we'll leave it at dtsttcpw for the moment.
+        if(@font.symbol?)
+          txt = enc.convert_symbol(txt)
+          if(UTF.match(@target_encoding))
+            @utf16_iconv ||= Iconv.new(@target_encoding.to_s, 'utf16be')
+            txt = @utf16_iconv.iconv(Symbol.to_utf16(txt))
+          end
+        elsif(tu = @font.to_unicode)
+          txt = tu.to_utf8(txt)
+          if(UTF.match(@target_encoding))
+            @utf8_iconv ||= Iconv.new(@target_encoding.to_s, 'utf8')
+            txt = @utf8_iconv.iconv(txt)
+          else
+            @symbol_iconv ||= Iconv.new('utf16be', 'utf8')
+            txt = Symbol.from_utf16(@symbol_iconv.iconv(txt))
+          end
+        end
+        txt
+      # FIXME: fix how encodings and Symbol font are handled
+      elsif(UTF.match(enc) && !UTF.match(@target_encoding) && @font.symbol?) 
+        @symbol_iconv ||= Iconv.new('utf16be', 'utf8')
+        txt = Symbol.from_utf16(@symbol_iconv.iconv(txt))
+      else
+        @iconv ||= Iconv.new(@target_encoding.to_s, enc.to_s)
+        @iconv.iconv(txt)
+      end
+    rescue NoMethodError, Iconv::InvalidEncoding, Iconv::IllegalSequence => e
+      txt
+		end
+		def same_column(other)
+			return false unless same_line(other)
+			if(other.is_a?(TextState))
+				testwidth = other.space_width * 2.0
+				width = @x - other.right_edge
+				width < testwidth
+			else
+				false
+			end
+		end
+		def same_line(other)
+			if(other.is_a?(TextState))
+				sy1, sy2 = [@y, self.y2].sort
+				oy1, oy2 = [other.y, other.y2].sort
+
+        pair = [[sy1, sy2], [oy1, oy2]].sort
+        overlap = pair[0][1] - pair[1][0]
+
+        [sy2 - sy1, oy2 - oy1].any? { |height|
+          # negative overlap means the lines don't touch
+          overlap / height > 0.4
+        }
+			else
+				false
+			end
+		end
+		def same_word(other)
+			return false unless same_line(other)
+			if(other.is_a?(TextState))
+				testwidth = other.space_width / 2.0
+				width = @x - other.x2
+				width < testwidth
+			else
+				false
+			end
+		end
+		def set_font(font)
+      @iconv = nil
+			@font = font
+		end
+		def set_font_size(size)
+			@font_size = size.to_f
+		end
+		def set_lead(lead)
+			@lead = lead.to_f
+		end
+		def set_char_spacing(line)
+			@char_spacing = line.to_f * USER_SPACE 
+		end
+		def set_txt(txt)
+			#call the unescape_txt method,
+			#so that \334 is replaced by char �
+			#otherwise the calculation of the string width is wrong!!!!
+			unescape_txt!(txt)
+			@boxwidth = 0
+      txt.rstrip.each_byte do |char|
+        @boxwidth += char_width(char)
+      end
+      @w = @boxwidth
+      if white = txt[/\s+$/u]
+        white.each_byte do |char|
+          @w += char_width(char)
+        end
+      end
+			@txt = recode_txt(txt)
+		end
+		def set_word_spacing(word_spacing)
+			@word_spacing = word_spacing.to_f * USER_SPACE 
+		end
+		def step
+      @dtmx = 0
+			@tmy -= lead
+		end
+		def update_x(x_val)
+      @dtmx = 0
+			@tmx += x_val.to_f
+		end
+		def update_y(y_val)
+      @dtmx = 0
+			@tmy -= y_val.to_f
+		end
+		def unescape_txt!(txt)
+			txt.gsub!(/\\([0-9]{3})/n) { |match| $1.oct.chr }
+		end
+		protected 
+		attr_writer :x
+	end
+  class NontextElement < PositionedElement
+    attr_accessor :current_page
+    attr_reader :x, :y, :x2, :y2, :text_state
+    def initialize
+      @x = @matrix_x = 0.0
+      @y = 0.0
+      @w = 0.0
+      @cmxscale = @tmxscale = 1.0
+      @cmyscale = @tmyscale = 1.0
+      @xscale = 1.0
+      @yscale = 1.0
+      @space_width = -1
+      super
+    end
+    def empty?
+      false
+    end
+    def fire_early_callbacks(previous, callback_handler)
+      if(previous)
+        unless(same_line(previous))
+          callback_handler.send_line_break
+        end
+        if(@font && previous.font != @font)
+          callback_handler.new_font(@font)
+        end
+      end
+    end
+    def same_column(other)
+      false
+    end
+    def text_state=(ts)
+      @media_box = ts.media_box
+      @font = ts.font
+      @text_state = ts.dup
+    end
+    def method_missing(name, *args, &block)
+      @text_state.send(name, *args, &block)
+    end
+  end
+  class HorizontalRule < NontextElement
+    alias :x2 :x
+    alias :y2 :y
+    def initialize(x, y, dm)
+      super()
+      self.transformation_matrix = dm
+      set_x(x)
+      set_y(y)
+    end
+    def send_content(previous, callback_handler)
+      if(previous && !same_line(previous))
+        callback_handler.send_hr
+      end
+    end
+    def same_line(other)
+      if(other.is_a?(HorizontalRule))
+        (other.y - @y).abs < 10
+      else
+        false
+      end
+    end
+    def <=> other
+      if other.is_a?(HorizontalRule) && same_line(other)
+        0
+      else
+        super
+      end
+    end
+  end
+  class ImagePlacement < NontextElement
+    attr_reader :resource
+    def initialize(resource, x, y, dm)
+      super()
+      case resource
+      when InlineImage
+        @xobject = resource
+      else
+        @resource = resource.downcase[1..-1].to_sym
+      end
+      self.transformation_matrix = dm
+      @x = x
+      @y = y - @cmyscale / 2
+    end
+    def image
+      xobject.image
+    end
+    def same_line(other)
+      @y == other.y
+    end
+    def send_content(previous, callback_handler)
+      if img = xobject
+        callback_handler.send_image self
+      end
+    end
+    def xobject
+      @xobject ||= @current_page.resources.xobject(@resource)
+    end
+  end
+end