rpdf2txt 0.8.2

data/lib/rpdf2txt/default_handler.rb

@@ -0,0 +1,352 @@
+#!/usr/bin/env ruby
+#
+#	Rpdf2txt -- PDF to Text Parser
+#	Copyright (C) 2004 Mike Walder, Raphael Waltert, Hannes Wyss
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Z�rich, Switzerland
+#	hwyss@ywesee.com,	mwalder@ywesee.com, rwaltert@ywesee.com
+#
+# PdfParser -- Rpdf2txt-- 14.11.2002 -- mwalder@ywesee.com, rwaltert@ywesee.com
+
+module Rpdf2txt
+	module SymbolMap
+		#this map is not complete!!
+		SYMBOL_ENTITIES = {
+      3     =>  65,
+			8804	=> 163,
+			8805	=> 179,
+			#alpha-omega
+			913		=>	65,
+			914		=>	66,
+			915		=>	71,
+			916		=>	68,
+			917		=>	69,
+			918		=>	90,
+			919		=>	72,
+			920		=>	81,
+			921		=>	73,
+			922		=>	75,
+			923		=>	76,
+			924		=>	77,
+			925		=>	78,
+			926		=>	88,
+			927		=>	79,
+			928		=>	80,
+			929		=>	82,
+			931		=>	83,
+			932		=>	84,
+			933		=>	85,
+			934		=>	70,	
+			935		=>	67,
+			936		=>	89,
+			945		=>	97,
+			946		=>	98,
+			947		=> 103,
+			948		=> 100,
+			949		=> 101,
+			950		=> 122,
+			951		=> 104,
+			952		=> 113,
+			953		=> 105,
+			954		=> 107,
+			955		=> 108,
+			956		=> 109,
+			957		=> 110,
+			958		=> 120,
+			959		=> 111,
+			960		=> 112,
+			961		=> 114,
+			963		=> 115,
+			964		=> 116,
+			965		=> 117,
+			966		=> 102,
+			967		=>	99,
+			968		=> 121,
+			969		=> 119,
+			#dot?
+			8901	=>	46,
+			#intersection (cap)
+			8745	=> 199,
+			#union (cup)
+			8746	=> 200,
+			#infinity
+			8734	=> 165,
+			#integral
+			8747	=> 166,
+			#partial differential
+			8706	=> 182,
+			#not equal to
+			8800	=> 185,
+			#equal
+			61		=> 186,
+			#almost equal to
+			8776	=> 187,
+			#superset of
+			8835	=> 201,
+			# superset or equal to
+			8839	=> 202,
+			#not a subset of
+			8836	=> 203,
+			#subset of
+			8834	=> 204,
+			#subset or equal to
+			8838	=> 205,
+			#element of
+			8712	=> 206,
+			#not an element of
+			8713	=> 207,
+			#n-ary product
+			8719	=> 213,
+			#radic
+			8730	=> 214,
+			#n-ary sum
+			8721	=> 229,
+		}
+	end
+	module DefaultHandler
+    def column_count
+    end
+    def column_width
+    end
+    def identify_columns?
+      false
+    end
+    def send_image(handle)
+    end
+		def new_font(font)
+		end
+		def new_fontsize(size)
+		end
+		def send_column
+		end
+		def send_colspan
+		end
+		def send_eof
+			@out
+		end
+		def send_flowing_data(data)
+			self.out << data
+		end
+    def send_hr
+    end
+		def send_line_break
+			self.out << "\n"
+		end
+		def send_paragraph
+		end
+		def send_page
+			self.out << "\n\n"
+		end
+		def out
+			@out ||= ""
+		end
+	end
+	class SimpleHandler 
+		include DefaultHandler
+		include SymbolMap
+		def initialize(io="")
+			@out = io
+		end
+	end
+	class ColumnHandler < SimpleHandler
+		def initialize(outstream="", padding=' ')
+			super(outstream)
+			@lines = []
+      @padding = padding
+			send_line_break
+		end
+    def column_widths
+			all_lengths = []
+      last_column = @lines.collect { |line| line.size }.push(1).max - 1
+			@lines.each { |line|
+				line.each_with_index { |column, idx|
+					if(column.is_a?(String))
+						column.rstrip!
+						if(line[idx.next] || idx == last_column)
+              length = column.length
+              all_lengths[idx] = [length, all_lengths[idx].to_i].max
+            end
+					end
+				}
+			}
+			max_lengths = []
+			@lines.each { |line|
+				line.each_with_index { |column, idx|
+					if(column.is_a?(String))
+						length = column.length
+            idx2 = idx.next
+            rm = 0
+            while(line[idx2].nil? && (max = all_lengths[idx2]))
+              rm += max
+              length -= max
+              idx2 += 1
+            end
+            max_lengths[idx] = [length, max_lengths[idx].to_i].max
+					end
+				}
+        line[last_column] ||= nil
+			}
+      max_lengths
+    end
+    def identify_columns?
+      true
+    end
+		def send_column
+			@current_column += @colspan
+			@colspan = 1
+			@columns[@current_column] ||= if(@padding.respond_to?(:foldcase))
+                                      u(@padding.dup)
+                                    else
+                                      @padding.dup
+                                    end
+		end
+		def send_colspan
+			@colspan += 1
+		end
+    def send_hr
+			@columns << :hr
+    end
+    def send_image(image)
+      @columns << :image
+    end
+		def send_flowing_data(data)
+			@columns[@current_column] << data
+		end
+		def send_line_break
+			@columns = []
+			@lines.push(@columns)
+			@current_column = -1
+			@colspan = 1
+		end
+		def send_page
+      max_lengths = column_widths
+			@lines.each { |line|
+				line.each_with_index { |column, idx|
+					if(column)
+						idx2 = idx.next
+						accumulated = max_lengths[idx].to_i
+						while((pad = max_lengths[idx2]) && !line[idx2])
+							accumulated += pad
+							idx2 += 1
+						end
+            case column
+            when :image
+              @out << " #IMAGE# ".ljust(accumulated)
+            when :hr
+              @out << @padding.dup.ljust(accumulated, '-')
+            else
+              @out << column.ljust(accumulated)
+            end
+					end
+				}
+				@out << "\n"
+			}
+			@lines.clear
+			send_line_break
+			super
+		end
+	end
+	class RecordingHandler
+		def initialize(out = $stdout, columns=false)
+			require 'yaml'
+			@out = out
+			@out << "require 'yaml'\n"
+      @columns = columns
+		end
+    def identify_columns?
+      @columns
+    end
+		def new_font(font)
+			@out << <<-EOS
+font = YAML.load <<-EOF
+#{font.to_yaml}
+EOF
+@writer.new_font(font)
+			EOS
+		end 
+		def method_missing(symbol, *args, &block)
+			argstr = args.collect { |arg| arg.inspect }.join(', ')
+			@out << <<-EOS
+@writer.#{symbol}(#{argstr})
+			EOS
+		end
+	end
+	class HTMLHandler 
+		include DefaultHandler
+		include SymbolMap
+		def initialize
+			super
+			@state = {
+				:italic	=> false,
+				:bold		=> false,
+				:font		=> false,
+			}
+		end
+		def new_font(font)
+			unless(font.nil?)
+				if(font.bold?)
+					if(!@state[:bold])
+						self.out << "<b>"
+						@state[:bold] = true
+					end
+				else
+					if(@state[:bold])
+						self.out << "</b>"
+						@state[:bold] = false
+					end
+				end
+				if(font.italic?)
+					if(!@state[:italic])
+						self.out << "<i>"
+						@state[:italic] = true
+					end
+				else
+					if(@state[:italic])
+						self.out << "</i>"
+						@state[:italic] = false
+					end
+				end
+				if(@state[:font])
+					self.out << "</font>"
+					@state[:font] = false
+				end
+				if(@state[:pre])
+					self.out << "</pre>"
+					@state[:pre] = false
+				end
+				font_name = font.basefont_name
+				if(!/symbol/i.match(font_name).nil?)
+					self.out << "<font face=\"Symbol\">" 
+					@state[:font] = true
+				end
+				if (!/courier/i.match(font_name).nil?)
+					self.out << "<pre>"
+					@state[:pre] = true
+				end
+			end
+		end
+		def send_line_break
+			@out << "<br>"
+		end
+		def send_paragraph
+			self.out << "<p>"
+		end
+		def send_page
+			self.out << "<p>"
+		end
+	end
+end

data/lib/rpdf2txt/lzw.rb

@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+# LZW -- rpdf2txt -- 09.07.2008 -- hwyss@ywesee.com
+
+module Rpdf2txt
+  class LZW
+    CLEAR = 256
+    EOD = 257
+    def self.decode data, early_change=1
+      self.new(early_change).decode data
+    end
+    def initialize(early_change=1)
+      @early_change = early_change
+      @__dict = (0..255).collect { |num| num.chr }.push :clear, :eod
+      init_dictionary
+    end
+    def init_dictionary
+      @dictionary = @__dict.dup
+      @code_length = 9
+      @boundary = 512 - @early_change
+    end
+    def decode data
+      bits, = data.unpack('B*')
+      expected_codes = bits.size / 12
+      code = old_code = 0
+      result = ''
+      max = codes = clears = 0
+      while(!bits.empty? && (code = get_next_code bits) && code != EOD)
+        codes += 1
+        if code == CLEAR
+          clears +=1 
+          init_dictionary
+          code = get_next_code bits
+          if code.nil? || code == EOD 
+            return result
+          end
+          result << @dictionary[code]
+          old_code = code
+        else
+          if string = @dictionary[code]
+            result << string
+            update_dictionary @dictionary[old_code] + string[0,1]
+            old_code = code
+          elsif code == @dictionary.size
+            string = @dictionary[old_code]
+            string += string[0,1]
+            result << string
+            update_dictionary string
+            old_code = code
+          else
+            raise 'Bad compressed code: %s' % code
+          end
+        end
+        max = [max, result.size].max
+      end
+      result
+    end
+    def get_next_code bits
+      chunk = bits.slice!(0, @code_length).to_i(2)
+    end
+    def update_dictionary(str)
+      @dictionary.push str
+      if @dictionary.size >= @boundary && @code_length < 12
+        @code_length += 1
+        @boundary = (2**@code_length - @early_change)
+      end
+      str
+    end
+  end
+end

data/lib/rpdf2txt/object.rb

@@ -0,0 +1,1114 @@
+#!/usr/bin/env ruby
+#
+#	Rpdf2txt -- PDF to Text Parser
+#	Copyright (C) 2003 Andreas Schrafl, Hannes Wyss
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+#
+#	ywesee - intellectual capital connected, Winterthurerstrasse 52, CH-8006 Zürich, Switzerland
+#	hwyss@ywesee.com,	aschrafl@ywesee.com
+#
+# PdfObject -- Rpdf2txt -- 21.11.2002 -- aschrafl@ywesee.com
+
+require 'zlib'
+require 'rpdf2txt/text'
+require 'rpdf2txt/attributesparser'
+require 'rpdf2txt/cmapparser'
+require 'rpdf2txt/symbol'
+require 'md5'
+require 'matrix'
+
+module Rpdf2txt
+	class PdfObject
+		attr_reader :attributes
+		attr_accessor :decoder, :src, :oid
+		def initialize(src=nil, target_encoding='utf8')
+			@attributes = {}
+			@src = src
+			@target_encoding = target_encoding
+			parse_attributes() unless @src.nil?
+		end
+		def build_tree(object_catalogue, parent=nil)
+			@attributes.each { |key, value|
+        case value
+        when Array
+          value.collect! { |obj| catalogue_object(object_catalogue, obj) || obj }
+        when String
+          if obj = catalogue_object(object_catalogue, value)
+            @attributes.store(key, obj)
+          end
+				end
+			}
+		end
+    def catalogue_object(catalogue, reference)
+      if reference.is_a?(String) && (match = /^(\d+)\s+\d+\s+R/n.match reference)
+        catalogue[match[1].to_i]
+      end
+    end
+		def decoded_stream
+			raise "abstract method decoded_stream called in #{self.class}; built from source: \n #{@src.tr("\r", "\n")}"
+		end
+		def oid
+			@oid ||= extract_oid(@src)
+		end
+		def revision_id
+			@revision_id ||= extract_revision_id(@src)
+		end
+		def extract_attribute_stream
+			lastindex = @src.index('stream') || -1
+			index = @src.index('<<')
+			rindex = @src.rindex('>>', lastindex)
+			@src[index..(rindex+1)] if(index && rindex)
+		end
+		def _parse_attributes(src)
+			Rpdf2txt.attributes_parser.parse(src)
+		end
+		def parse_attributes
+			src = self.extract_attribute_stream
+			if(src.nil?)
+				@attibutes = {}
+			else
+				ast = self._parse_attributes(src)
+				ast.compact!
+				@attributes = extract_attributes(ast)
+			end
+		end	
+		private
+		def extract_oid(string)
+			/^\d+/n.match(string).to_s.to_i
+		end
+		def extract_revision_id(string)
+			/\s\d+/n.match(string).to_s.to_i
+		end
+		def extract_attributes(ast)
+			if(ast.children_names.include?('value'))
+				pdf_unescape(ast.value)
+			elsif(ast.children_names.include?('text'))
+				pdf_unescape(ast.text.value[1...-1])
+			elsif(ast.children_names.include?('values'))
+				ast.values.collect { |child| extract_attributes(child) }
+			elsif(ast.children_names.include?('pairs'))
+				result = {}
+				ast.pairs.each { |pair| 
+					k, v = pair
+					keystr = k.value.strip.tr('/','')
+					unless(keystr.empty?)
+						result.store(keystr.downcase.intern, extract_attributes(v))
+					end
+				}
+				result
+			else
+				value = ast
+			end
+		end
+    def pdf_escape(input)
+      input.gsub(/\\/, '\\\\').gsub(/\n/n, '\n')\
+        .gsub(/\r/n, '\r').gsub(/[()]/n, '\\&')
+    end
+    def pdf_unescape(input)
+      input.gsub(/\\n/n, "\n").gsub(/\\r/n, "\r").\
+        gsub(/\\\)/n, ')').gsub(/\\\(/n, '(').gsub(/\\\\/n, '\\')
+    end
+	end
+	class PdfEncrypt < PdfObject
+		class DecryptionError < RuntimeError
+		end
+		PADDING = "\x28\xBF\x4E\x5E\x4E\x75\x8A\x41\x64\x00\x4E\x56\xFF\xFA\x01\x08\x2E\x2E\x00\xB6\xD0\x68\x3E\x80\x2F\x0C\xA9\xFE\x64\x53\x69\x7A"
+    def arc4(key, input)
+      output = ''
+      s, j, k = (0..255).to_a, 0, (key*256)[0,256].unpack('C*')
+      (0..255).each { |x|
+        j = (j + s[x] + k[x]) % 256
+        s[x], s[j] = s[j], s[x]
+      }
+      i = j = 0
+      input.each_byte { |b|
+        i = (i + 1) % 256
+        j = (j + s[i]) % 256
+        s[i], s[j] = s[j], s[i]
+        output << (b ^ s[(s[i] + s[j])%256]).chr
+      }
+      output
+    end
+    def compute_user_key encryption_key
+      if revision < 3
+        pdf_escape arc4(encryption_key, PADDING)
+      else
+        crypt = Digest::MD5.digest PADDING + file_id
+        20.times do |xor|
+          key = encryption_key.unpack('C*').collect! do |byte|
+            byte ^ xor
+          end.pack('C*')
+          crypt = arc4(key, crypt)
+        end
+        pdf_escape crypt
+      end
+    end
+		def decrypt(pdf_object)
+			arc4_key = decrypt_key(pdf_object)
+			stream = pdf_object.raw_stream
+			arc4(arc4_key, stream)
+		end
+		def decrypt_key(pdf_object)
+			oid = pdf_object.oid
+			rev_id = pdf_object.revision_id
+			#if it is a ppc we use reverse
+			if(self.big_endian?)
+				oid_three_bytes = [oid].pack('I*').reverse[0,3]
+				rev_id_two_bytes = [rev_id].pack('I*').reverse[0,2]
+			else
+				oid_three_bytes = [oid].pack('I*')[0,3]
+				rev_id_two_bytes = [rev_id].pack('I*')[0,2]
+			end
+			input = encryption_key << oid_three_bytes << rev_id_two_bytes
+			digest = Digest::MD5.digest(input)
+			digest[0,[keylength + 5,16].min]
+		end
+		def big_endian?
+			#big endian (ppc) little endian x86
+			if ([1].pack('I*') ==  "\000\000\000\001") 
+				true 
+			else 
+				false
+			end
+		end
+    def encryption_key
+      input_string = PADDING.dup
+      ## we don't support a user-password. if we did, it would have to replace
+      #  the first [n..32] bytes of the padding string here.
+      input_string << owner_key
+      input_string << permission_flag
+      input_string << file_id
+      ## revision >= 4: add 0xffffffff if document metadata is not encrypted
+      digest = Digest::MD5.digest(input_string)
+      uk = user_key
+      if revision >= 3
+        50.times do digest = Digest::MD5.digest(digest[0,keylength]) end
+        uk = uk[0,16]
+      end
+      encryption_key = digest[0,keylength]
+      test_key = compute_user_key encryption_key
+      if(test_key != uk)
+        raise DecryptionError, "test-key did not match user-key ('#{test_key.inspect}' / '#{uk.inspect}')"
+      end
+      encryption_key
+    end
+		def file_id= (file_id)
+			@file_id = file_id
+		end
+    def file_id
+      [@file_id].pack("H*")
+    end
+    def keylength
+      @keylength ||= (@attributes[:length] || 40).to_i / 8
+    end
+		def owner_key
+			@attributes[:o].to_s
+		end
+		def permission_flag
+			if (self.big_endian?)
+				[@attributes[:p].to_i].pack('I*').reverse
+			else
+				[@attributes[:p].to_i].pack('I*')
+			end
+		end
+    def revision
+      @attributes[:r].to_i
+    end
+    def user_key
+      @attributes[:u].to_s
+    end
+	end
+  class Encoding < PdfObject
+    def differences
+      @differences or
+      begin
+        @differences = {}
+        offset = 0
+        idx = 0
+        @attributes[:differences].each { |diff|
+          if(/^\d+$/n.match(diff))
+            offset = diff.to_i
+          else
+            @differences.store(offset + idx, diff[/\w+/n])
+            idx += 1
+          end
+        }
+        @differences
+      end
+    end
+    def convert_symbol(txt)
+      res = ''
+      txt.each_byte { |byte|
+        if(name = differences[byte])
+          byte = Symbol.byte(name) || byte
+        end
+        res << byte
+      }
+      res
+    end
+    def symbol_name(byte)
+      differences[byte]
+    end
+  end
+	class Font < PdfObject
+		attr_accessor :cmap, :descriptor, :rendering_mode, :skewed
+		ENCODINGS = {
+      '/Identity-H'       =>  'ascii',
+			'/MacRomanEncoding'	=>	'mac',
+      '/UTF8'             =>  'utf8',
+			'/WinAnsiEncoding'	=>	'ms-ansi',
+		}
+    AFM_PTRN = /^C\s*(\d+)\s*;\s*
+                WX\s*(\d+)\s*;\s*
+                N\s*(\w+)/xn
+		def basefont_name
+			@attributes[:basefont]
+		end
+    def basefont_width(char)
+      basefont_widths[char]
+    end
+    def basefont_widths
+      @basefont_widths ||= load_basefont_widths
+    end
+		def bold?
+			(!!/bold/in.match(basefont_name.to_s)) || @rendering_mode == "2"
+		end
+    def build_tree(object_catalogue, parent = nil)
+      super
+      if(desc = @attributes[:descriptor])
+        @descriptor = desc
+        desc.build_tree(object_catalogue, self)
+      end
+    end
+		def encoding
+      enc = @attributes[:encoding]
+      if(enc.is_a?(Encoding))
+        enc
+      else
+        ENCODINGS.fetch(enc) { 
+          warn "unknown encoding #{enc}"
+          enc
+        }
+      end
+		end
+		def italic?
+			@skewed || !!/italic/in.match(basefont_name.to_s)
+		end
+		def width(char)
+			if(char.is_a?(String) && char.length == 1)
+				char = char[0]
+			end
+      _width(char) || named_width(char)
+		end
+    def widths
+      @widths ||= (@attributes[:widths] || [])
+    end
+    def symbol?
+      !!/symbol/in.match(basefont_name.to_s)
+    end
+    def to_unicode
+      @to_unicode ||= (tu = @attributes[:tounicode]) && tu.to_cmap
+    end
+		private
+		def first_char
+			@attributes[:firstchar].to_i
+		end
+    def load_basefont_widths
+      widths = {}
+      path = File.join(File.dirname(__FILE__), 'data', 'fonts', 
+                       "%s.afm" % basefont_name)
+      if(File.readable?(path))
+        File.read(path).scan(AFM_PTRN) { |char, width, name|
+          widths.store(char.to_i, width)
+          widths.store(name, width)
+        }
+      end
+      widths
+    end
+    def named_width(char)
+      enc = @attributes[:encoding]
+      if(enc.is_a?(Encoding))
+        _width(enc.symbol_name(char))
+      end
+    end
+    def _width(char)
+      width = widths.at(char - first_char) if(char.is_a? Integer)
+			width ||= basefont_width(char)
+      width.to_i if(width)
+    end
+	end
+	class FontDescriptor < PdfObject
+	end
+	class Unknown < PdfObject
+    def to_i(*args)
+      ((match = /obj\s*(\d+)/n.match(@src)) && match[1]).to_i(*args)
+    end
+	end	
+	class Resource < PdfObject
+		def initialize(src=nil, target_encoding='utf8')
+			if(src.is_a? Hash)
+				@attributes = src
+			elsif(src.is_a? String)
+				super
+			else
+				@attributes = {}
+			end
+			@fonts = {}
+      @xobjects = {}
+		end
+		def build_tree(object_catalogue, parent=nil)
+			super
+      [:font, :xobject].each { |type| build_attributes type, object_catalogue }
+		end
+		def font(key)
+			@fonts[key]
+		end
+    def xobject(key)
+      @xobjects[key]
+    end
+    private
+    def build_attributes(type, object_catalogue)
+      storage = instance_variable_get "@#{type}s"
+      attribute = @attributes[type]
+      if attribute.is_a?(Hash)
+        _build_attributes(storage, attribute, object_catalogue)
+      elsif attribute.is_a?(PdfHash)
+        _build_attributes(storage, attribute.contents, object_catalogue)
+      end
+    end
+    def _build_attributes(storage, hash, object_catalogue)
+      hash.each { |key, val|
+        oid = /^\d+/n.match(val).to_s.to_i
+        if(obj = object_catalogue[oid])
+          obj.build_tree(object_catalogue)
+        end
+        storage.store(key, obj)
+      }
+    end
+	end
+	class TrailerDictionary < PdfObject
+		def file_id
+			/[a-zA-Z0-9]+/n.match(@attributes[:id].first).to_s
+		end
+		def encrypt_id
+			extract_oid(@attributes[:encrypt])
+		end
+=begin
+		def parse_attributes
+			index = @src.index('trailer')
+			rindex = @src.rindex('startxref')
+			unless(index && rindex)
+				@attibutes = {}
+			else
+				#set correct offsets <<(begin)  >>(end)
+				src = @src[index+7..(rindex-1)]
+				ast = Rpdf2txt.attributes_parser.parse(src)
+				ast.compact!
+				@attributes = extract_attributes(ast)
+			end
+		end
+=end
+		def root_id
+			extract_oid(@attributes[:root])
+		end
+		def update(trailer_dict)
+			@attributes.update(trailer_dict.attributes)
+		end
+		protected
+		attr_reader :attributes
+	end
+	class TreeNode < PdfObject
+		include Enumerable
+		attr_reader :parent
+		def build_tree(object_catalogue, parent=nil)
+			super
+			@parent = parent
+			self
+		end
+		def each
+			yield self
+		end
+		def extract_oids(array)
+			array.collect{ |dirty_id|
+				if(match = /\d+/on.match(dirty_id))
+					match[0].to_i
+				end
+			}.compact
+		end
+		def root?
+			!(@parent || @attributes[:parent])
+		end
+	end	
+	class CatalogNode < TreeNode
+		def build_tree(object_catalogue, parent=nil)
+			id = extract_oids(@attributes[:pages]).first
+			@pages = object_catalogue[id]
+			@pages.build_tree(object_catalogue, self)
+			super
+		end
+		def each(&block)
+			@pages.each(&block)
+		end
+	end
+	class PageNode < TreeNode
+		attr_reader :kids
+		def build_tree(object_catalogue, parent=nil)
+			@kids = []
+			extract_oids(@attributes[:kids]).each { |id|
+				child = object_catalogue[id]
+				@kids.push(child)
+				child.build_tree(object_catalogue, self)
+			}
+			super
+		end
+		def each
+			@kids.each { |kid|
+				kid.each { |result| yield result }
+			}
+		end
+		def media_box
+			if(mb = @attributes[:mediabox])
+				mb.collect { |val| val.to_f }
+			end
+		end
+	end
+	class PageLeaf < TreeNode
+		attr_reader :contents, :resources
+		def initialize(*args)
+			super
+			@text_state = TextState.new(@target_encoding)
+		end
+		def build_tree(object_catalogue, parent=nil)
+			@contents=[]
+			extract_oids(@attributes[:contents]).each{ |id|
+				content = object_catalogue[id]
+				@contents.push(content)
+				content.build_tree(object_catalogue, self) if content.respond_to?(:build_tree)
+			} 
+			resources = @attributes[:resources]
+			if(resources.is_a? String)
+				@resources = object_catalogue[extract_oids([resources]).first]
+			elsif resources.is_a? Hash
+				@resources = Resource.new(resources)
+			elsif(resources.nil? && @parent)
+				@resources = @parent.resources
+			else
+				@resources = Resource.new()
+			end
+			@resources.build_tree(object_catalogue) if @resources.is_a? Resource
+			super
+		end
+		def font(key)
+			@resources.font(key)
+		end
+		def media_box
+			if(parent)
+				parent.media_box
+			end
+		end
+		def text(callback_handler)
+			concat_stream = Stream.new('')
+			if(@contents.size == 1 && @contents.first.is_a?(ReferenceArray))
+				@contents.first.build_stream(concat_stream)
+			else
+				@contents.each { |stream|
+					concat_stream.append(stream.decoded_stream)
+				}
+			end
+			@text_state.media_box = self.media_box
+			text_snippets = concat_stream.extract_text_objects(self, @text_state)
+      join_snippets(text_snippets, callback_handler)
+		end
+		private
+    def each_pair(text_snippets, &block)
+			text_snippets.inject(nil) { |last_text_state, text_state|
+        if text_state.whitespace_overlap?(last_text_state)
+          last_text_state
+        else
+          block.call(last_text_state, text_state)
+          text_state
+        end
+			}
+    end
+		def identify_columns(text_snippets, hints={})
+      ## find narrowest space on page and use as grid-width
+      space = text_snippets.collect { |snip| 
+        snip.space_width }.select { |w| w > 0 }.min || 100.0
+      ## count ocurrences of snippets in each grid-column
+			positions = {}
+			each_pair(text_snippets) { |last_text_state, text_state|
+        unless(text_state.empty? \
+               || (last_text_state \
+                   && text_state.same_column(last_text_state) \
+                   && !last_text_state.empty?))
+          idx = ((text_state.x) / space).floor
+          positions[idx] = positions[idx].to_i + 1
+        end
+			}
+      return [] if(positions.empty?)
+      ## find the average distance between candidate columns
+      #previous = 0
+      total = 0
+      sorted = positions.sort
+      begin
+        previous, count = sorted.shift
+      end until(count.nil? || count > 2)
+      counts = []
+      width = sorted.inject(0) { |sum, (pos,count)| 
+        counts.push [count,pos]
+        if(count > 2)
+          total += 1
+          sum += (pos - previous)
+          previous = pos
+        end
+        sum
+      }
+      if (colcount = hints[:count]) && counts.size >= colcount
+        res = counts.sort[-colcount..-1].collect do |count, pos|
+          pos * space
+        end
+        return res.sort
+      end
+
+      cutwidth = hints[:width]
+      if cutwidth.is_a?(String)
+        dividend, divisor = cutwidth.split '/', 2
+        cutwidth = width * dividend.to_f / divisor.to_f
+      end
+      cutwidth ||= (total.nonzero?) ? width / total * 0.9 : width
+      ## select probable columns
+      previous = -cutwidth
+			res = []
+      sorted = positions.sort
+      offset, _ = sorted.first
+      sorted.each_with_index { |(pos, count), idx| 
+        ndx = idx.next
+        pos -= offset
+        begin
+          nxtpos, nxtcount = sorted[ndx]
+          ndx += 1
+        end until(nxtcount.nil? || nxtcount > 3)
+        nxtpos -= offset if nxtpos
+        if((count > 1 && (pos - previous) > cutwidth) \
+           && !(nxtcount.to_i > count && (nxtpos - pos) < cutwidth))
+          previous = pos
+          res.push pos + offset
+        end
+      }
+      res.collect { |pos| pos * space }.sort
+		end
+		def join_snippets(text_snippets, callback_handler)
+			text_snippets.sort!
+			columns = []
+      if(callback_handler.identify_columns?)
+        columns = identify_columns(text_snippets,
+                                   :width => callback_handler.column_width,
+                                   :count => callback_handler.column_count)
+        columns.shift #throw away the first colum - we'll use the left media-edge
+      end
+			next_column = nil
+			working_set = []
+			each_pair(text_snippets) { |last_text_state, text_state|
+				text_state.fire_early_callbacks(last_text_state, 
+                                        callback_handler)
+        # are we on a new line?
+				unless(last_text_state && text_state.same_line(last_text_state))
+					working_set = columns.dup
+					next_column = working_set.shift
+					last_text_state = nil
+					callback_handler.send_column
+				end
+        x2 = last_text_state && last_text_state.right_edge.to_i
+				while(next_column && (text_state.x.to_i >= next_column.to_i))
+					if(x2 && (x2 > next_column.to_i) && !last_text_state.empty?)
+						callback_handler.send_colspan
+					else
+						callback_handler.send_column
+					end
+					next_column = working_set.shift
+				end
+        text_state.send_content(last_text_state, callback_handler)
+			}
+		end
+	end
+	class Stream < PdfObject
+		num = "([0-9.-]+)\\s*"
+		dm_str = "#{num}#{num}#{num}#{num}#{num}#{num}cm\\b"
+    xobj = '(/\S+)\s*(\bDo\b)'
+		@@nontext_scan_pattern = %r!(?:#{dm_str})|(\b[qQ]\b)|#{xobj}|\bBI\b(.*?)\bID\b(.*?)\b(EI)\b!mn
+		@@hr_scan_pattern = /#{num}#{num}(\b[lm]\b)/mn
+    BT_PATTERN = /\bBT\b(?!(\\[()]|[^(\\])*\))/mn
+    ET_PATTERN = /\bET\b(?!(\\[()]|[^(\\])*\))/mn
+    FAIL_PTRN = /\((\\[()]|[^)])*\bET\b\s*$/mn
+		def append(decoded_stream)
+			(@decoded_stream  ||= '') << decoded_stream
+		end 
+		def decoded_stream=(decoded_stream)
+			@decoded_stream = decoded_stream
+		end
+		def decoded_stream
+			@decoded_stream ||= decode_raw_stream
+		end
+    def extract_horizontal_rules(dm_src, dmatrix, result)
+      last_x = 0
+      last_y = 0
+      dm_src.scan(@@hr_scan_pattern) { |matches|
+        case matches.last.to_s[-1]
+        when ?l
+          x = matches[0].to_f
+          y = matches[1].to_f
+          if(x != last_x && y == last_y)
+            hr = HorizontalRule.new(x, y, dmatrix)
+            hr.current_page, hr.text_state = @page, @text_state
+            result.push(hr)
+          end
+          last_x = x
+          last_y = y
+        when ?m
+          last_x = matches[0].to_f
+          last_y = matches[1].to_f
+        end
+      }
+    end
+    def extract_nontext_objects(dm_src, dmatrix, stack, result)
+      dm_src.scan(@@nontext_scan_pattern) { |matches|
+        matches = matches.compact
+        case matches.last
+        when 'q'
+          stack.push(dmatrix)
+        when 'Q'
+          dmatrix = stack.pop
+        when 'Do'
+          x, y = (txt = result.last) ? [txt.x, txt.y] : [0, 0]
+          ip = ImagePlacement.new(matches[-2], x, y, dmatrix)
+          ip.current_page, ip.text_state = @page, @text_state
+          result.push ip
+        when 'EI'
+          attrs, data, _ = matches
+          im = InlineImage.new attrs, data.strip
+          ip = ImagePlacement.new im, 0, 0, dmatrix
+          ip.current_page, ip.text_state = @page, @text_state
+          result.push ip
+        else
+          mmatrix = Matrix[[matches[0].to_f, matches[1].to_f,0],
+                           [matches[2].to_f, matches[3].to_f,0],
+                           [matches[4].to_f, matches[5].to_f,1]]
+          dmatrix = dmatrix * mmatrix
+        end
+      }
+      dmatrix
+    end
+		def extract_text_objects(page, text_state)
+      @page, @text_state = page, text_state
+			stack = []
+			result = []
+			startpoint = decoded_stream.index(BT_PATTERN)
+			endpoint = decoded_stream.index(ET_PATTERN)
+      while FAIL_PTRN.match(decoded_stream[0..(endpoint+2)])
+        endpoint = decoded_stream.index(ET_PATTERN, endpoint.next)
+      end
+			unless(startpoint && endpoint && (startpoint < endpoint))
+				startpoint = 0
+			end
+      rotation = (page && Math::PI * page.attributes[:rotate].to_f / 180) || 0
+			dmatrix = Matrix[[Math.cos(rotation),Math.sin(rotation),0],
+                       [Math.sin(rotation),-Math.cos(rotation),0],
+                       [0,0,1]]
+
+			dm_src = decoded_stream[0...startpoint]
+			while(endpoint && startpoint)
+				### pick out the bits in between Text that are relevant to 
+				### text positioning (such as the device-transformation-matrix)
+				### NOTE: as far as I understand, the device matrix should 
+				###       not be used to position text. However it is used 
+				###       by some PDF-Creators and therefore we have to include
+				###       it in our calculations.
+        dmatrix = extract_nontext_objects(dm_src, dmatrix, stack, result)
+        extract_horizontal_rules(dm_src, dmatrix, result)
+				tsrc = decoded_stream[startpoint..(endpoint+2)]
+        while FAIL_PTRN.match(tsrc)
+          endpoint = decoded_stream.index(ET_PATTERN, endpoint + 2) || -1
+          tsrc = decoded_stream[startpoint..(endpoint+2)]
+        end
+				text = Text.new(tsrc, @target_encoding, dmatrix)
+				text.current_page = page
+				text.text_state = text_state
+        result.concat text.scan
+				startpoint = decoded_stream.index(BT_PATTERN, endpoint)
+				if(startpoint)
+					dm_src = decoded_stream[endpoint...startpoint]
+					endpoint = decoded_stream.index(ET_PATTERN, startpoint)
+				end
+			end
+			result
+		end
+		def raw_stream
+			@raw_stream ||= @src.scan(/stream[\r\n]{1,2}(.*)endstream/mn).to_s
+		end
+		def decode_raw_stream
+			@decrypted_stream = raw_stream
+			unless(@decoder.nil?)
+				@decrypted_stream = @decoder.decrypt(self)
+			end
+			stream = @decrypted_stream
+			[@attributes[:filter]].flatten.compact.each { |filter|
+        begin
+          stream = case filter
+                   when "/FlateDecode"
+                     flate_decode stream
+                   when "/LZWDecode"
+                     lzw_decode stream
+                   else
+                     raise "Unimplemented filter: #{filter}"
+                   end
+        rescue StandardError => err
+          warn "'#{err.message}' when filtering with #{filter}"
+        end
+			}
+			stream
+		end
+    def flate_decode(data)
+      Zlib::Inflate.inflate(data)
+    end
+    def lzw_decode(data)
+      require 'rpdf2txt/lzw'
+      earlychange = (parm = @attributes[:decodeparms]) && parm[:earlychange]
+      if length = @attributes[:length]
+        data = data[0, length.to_i]
+      end
+      LZW.decode data, (earlychange || 1).to_i
+    end
+		def to_cmap
+			cmap = CMap.new(@src, @target_encoding)
+		end
+	end
+  class ObjStream < Stream
+  end
+  class Image < Stream
+    COLORMAPS = {
+      '/DeviceRGB'  => 'RGB',
+      '/DeviceGray' => 'I',
+      '/DeviceCMYK' => 'CMYK',
+    }
+    def image
+      require 'RMagick'
+      @image or begin
+        columns = @attributes[:width].to_i
+        rows = @attributes[:height].to_i
+        depth = @attributes[:bitspercomponent].to_i
+        mask = @attributes[:mask]
+        color_grades = 2 ** depth - 1
+        colorspace, basespace, index_colors, index = @attributes[:colorspace]
+        index_colors = index_colors.to_i
+        colormap = COLORMAPS[colorspace] || COLORMAPS[basespace] || 'RGB'
+        colors = colormap.length
+        pixels = extract_pixels(decoded_stream, depth)
+        case colorspace
+        when '/Indexed'
+          ## FIXME: this works for some images, but seems to be wrong
+          #         according to the Documentation
+          if mask.is_a?(Array) && (pixels.size - 1 ) > rows * columns
+            range = (mask[0].to_i)..(mask[1].to_i)
+            pixels.delete_if { |idx| range.include? idx }
+          end
+          # for indexed images, index_colors correctly describes the 
+          # depth of the resulting pixels, whereas bitspercomponent
+          # may not be accurate
+          color_grades = index_colors
+          map = extract_colormap(index, index_colors)
+          tmp = Array.new(pixels.size * colors)
+          pos = 0
+          pixels.each { |idx|
+            tmp[pos, colors] = map[idx * colors, colors]
+            pos += colors
+          }
+          pixels = tmp
+        end
+        ## this seems to be undocumented: PNG-images need to be decoded.
+        #  we can detect this by the additional Byte per Row:
+        if pixels.size == (columns * colors + 1) * rows
+          pixels = idat_decode pixels, columns, colors
+        elsif pixels.size > (rows * columns * colors)
+          pixels = pixels[0, rows * columns * colors]
+        end
+        if color_grades != (2 ** Magick::QuantumDepth - 1)
+          div = color_grades.to_f
+          pixels.collect! { |px| px / div }
+        end
+        @image = Magick::Image.constitute(columns, rows, colormap, pixels)
+      end
+    end
+    def idat_decode(data, width, colors)
+      scanline_length = colors * width + 1 # for filter
+      byte_width = width * colors
+
+      pixels = []
+      row = 0
+      until data.empty? do
+        row_data = data.slice! 0, scanline_length
+        filter = row_data.shift
+        case filter
+        when 0 then # None
+        when 1 then # Sub
+          row_data.each_with_index do |byte, index|
+            left = index < colors ? 0 : row_data[index - colors]
+            row_data[index] = (byte + left) % 256
+          end
+        when 2 then # Up
+          row_data.each_with_index do |byte, index|
+            upper = row == 0 ? 0 : pixels[ - byte_width + index ]
+            row_data[index] = (upper + byte) % 256
+          end
+        when 3 then # Average
+          row_data.each_with_index do |byte, index|
+            upper = row == 0 ? 0 : pixels[ - byte_width + index ]
+            left = index < colors ? 0 : row_data[index - colors]
+
+            row_data[index] = (byte + ((left + upper)/2).floor) % 256
+          end
+        when 4 then # Paeth
+          left = upper = upper_left = nil
+          row_data.each_with_index do |byte, index|
+
+            left = index < colors ? 0 : row_data[index - colors]
+            if row == 0 then
+              upper = upper_left = 0
+            else
+              upper_idx = - byte_width + index
+              upper = pixels[ upper_idx ]
+              upper_left = index < colors ? 0 : pixels[ upper_idx - colors ]
+            end
+
+            paeth = paeth left, upper, upper_left
+            row_data[index] = (byte + paeth) % 256
+          end
+        else
+          raise ArgumentError, "Invalid filter algorithm #{filter}"
+        end
+
+        pixels.concat row_data
+        row += 1
+      end
+      pixels
+    end
+    private
+    def extract_colormap(index, mask)
+      map = []
+      if index.is_a? Stream
+        map = index.decoded_stream.unpack('C*').collect { |int| int & mask }
+      elsif index.is_a? Array
+        map = index
+      end
+      map
+    end
+    def extract_pixels(stream, depth)
+      case depth
+      when 8
+        stream.unpack('C*')
+      else
+        tmp, = stream.unpack('B*')
+        pixels = []
+        tmp.scan(/.{#{depth}}/n) { |match|
+          pixels.push match.to_i(2)
+        }
+        pixels
+      end
+    end
+    def paeth(a, b, c) # left, above, upper left
+      p = a + b - c
+      pa = (p - a).abs
+      pb = (p - b).abs
+      pc = (p - c).abs
+
+      return a if pa <= pb && pa <= pc
+      return b if pb <= pc
+      c
+    end
+  end
+  class InlineImage < Image
+    ATTR_ABBREVIATIONS = {
+      :bpc => :bitspercomponent, :cs => :colorspace,
+      :d   => :decode,           :dp => :decodeparms,
+      :f   => :filter,           :h  => :height,
+      :im  => :imagemask,        :i  => :interpolate,
+      :w   => :width,
+    }
+    OTHER_ABBREVIATIONS = {
+      '/G'    => '/DeviceGray',
+      '/RGB'  => '/DeviceRGB',
+      '/CMYK' => '/DeviceCMYK',
+      '/I'    => '/Indexed',
+      '/AHx'  => '/ASCIIHexDecode',
+      '/A85'  => '/ASCII85Decode',
+      '/LZW'  => '/LZWDecode',
+      '/Fl'   => '/FlateDecode',
+      '/RL'   => '/RunLengthDecode',
+      '/CCF'  => '/CCITTFaxDecode',
+      '/DCT'  => '/DCTDecode',
+    }
+    def initialize(attrs, data)
+      super("<<" << attrs << ">>")
+      @raw_stream = data
+    end
+    def parse_attributes
+      super
+      ATTR_ABBREVIATIONS.each do |abbr, key|
+        if value = @attributes.delete(abbr)
+          @attributes.store key, OTHER_ABBREVIATIONS.fetch(value, value)
+        end
+      end
+    end
+  end
+	class CMap < Stream
+		attr_accessor :map
+		def initialize(*args)
+			@map = {}
+			super
+			parse_cmap()
+		end
+    def to_utf8(txt)
+      if(@map.nil?)
+        txt
+      elsif(txt.is_a?(Integer))
+        @map[txt]
+      else
+        txt.unpack('C*').collect { |byte| 
+          @map.fetch(byte, byte) }.pack('U*')
+      end
+    end
+    private
+		#bfchar definition
+		def add_to_map_bfchar(ast)
+			ast.compact!
+			ast.each { |child|
+				#convert in to decimal values
+				@map.store(_hexvalue(child.source), _hexvalue(child.target))
+			}
+			@map
+		end
+		#bfrange definition see page 457 of the pdf manual
+		def add_to_map_bfrange(ast)
+			ast.compact!
+			start_range = ast.start.value.to_s.hex
+			end_range = ast.stop.value.to_s.hex
+			if(ast.children_names.include?('explicit'))
+				explicit = ast.explicit
+				start_range.upto(end_range) {	|char|
+					@map.store(char, _hexvalue(explicit.shift))
+				}
+			else
+				offset = _hexvalue(ast.offset)
+				start_range.upto(end_range) {	|char|
+					@map.store(char, offset)
+					offset+=1
+				}
+			end
+			@map
+		end
+		def extract_bfchar
+			src = decoded_stream
+			unless(decoded_stream.index('beginbfchar').nil?)
+				index = decoded_stream.index('beginbfchar') + 12
+				rindex = decoded_stream.index('endbfchar')
+				src[index..(rindex)-1]
+			end
+		end
+		def extract_bfrange
+			src = decoded_stream
+			unless(decoded_stream.index('beginbfrange').nil?)
+				index = decoded_stream.index('beginbfrange') + 12
+				rindex = decoded_stream.index('endbfrange')
+				src[index..(rindex)-1]
+			end
+		end
+		def _hexvalue(ast)
+			ast.value.to_s.to_i(16)
+		end
+		def parse_cmap
+			if(src = extract_bfchar)
+				ast =	Rpdf2txt.cmap_parser.parse(src)
+				add_to_map_bfchar(ast)
+			end
+			if(src = extract_bfrange)
+				ast =	Rpdf2txt.cmap_range_parser.parse(src)
+				ast.each { |node|
+					add_to_map_bfrange(node)
+				}
+			end
+		end
+	end
+	class ReferenceArray < TreeNode
+		def build_tree(object_catalogue, parent=nil)
+			@contents=[]
+			@references.each{ |id|
+				@contents.push(object_catalogue[id]) if object_catalogue[id]
+			} 
+			super
+		end
+		def build_stream(concat_stream)
+			@contents.each { |stream|
+				concat_stream.append(stream.decoded_stream)
+			}
+			concat_stream
+		end
+		def parse_attributes
+			src = @src[@src.index('[')..@src.rindex(']')]
+			ast = _parse_attributes(src)
+			ast.compact!
+			@references = extract_oids(extract_attributes(ast))
+		end
+		def root?
+			false
+		end
+	end
+	class PdfArray < TreeNode
+		def build_tree(object_catalogue, parent=nil)
+			@contents=[]
+			super
+		end
+		def at(idx)
+			@contents.at(idx)
+		end
+		def each(&block)
+			@contents.each(&block)
+		end
+		def parse_attributes
+			src = @src[@src.index('[')..@src.rindex(']')]
+			ast = _parse_attributes(src)
+			ast.compact!
+			@contents = extract_attributes(ast)
+		end
+		def root?
+			false
+		end
+	end
+	class PdfHash < TreeNode
+    attr_reader :contents
+		def build_tree(object_catalogue, parent=nil)
+			@contents={}
+			super
+		end
+		def parse_attributes
+			src = @src[@src.index('<<')..@src.rindex('>')]
+			ast = _parse_attributes(src)
+			ast.compact!
+			@contents = extract_attributes(ast)
+		end
+		def root?
+			false
+		end
+	end
+end