pdf-reader 1.1.1 → 1.2.0

data/CHANGELOG

@@ -1,3 +1,11 @@
+v1.2.0 (28th AUgust 2012)
+- Feature: correctly extract text using surrogate pairs and ligatures
+  (thanks Nathaniel Madura)
+- Speed optimisation: cache tokenised Form XObjects to avoid re-parsing them
+- Feature: support opening documents with some junk bytes prepended to file
+  (thanks Paul Gallagher)
+  - Acrobat does this, so it seemed reasonable to add support
+
 v1.1.1 (9th May 2012)
 - bugfix release to improve parsing of some PDFs
 

data/bin/pdf_text

@@ -1,8 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-$LOAD_PATH.unshift(File.dirname(__FILE__) + "/../lib")
-
 require 'pdf/reader'
 
 if ARGV.empty?

data/examples/extract_images.rb

@@ -86,14 +86,15 @@ module ExtractImages
       tiff = header.dup
       tiff << short_tag.call( 256, 1, w ) # image width
       tiff << short_tag.call( 257, 1, h ) # image height
-      tiff << long_tag.call( 258, 4, (header.size + (tag_count*12))) # bits per pixel
+      tiff << long_tag.call( 258, 4, (header.size + (tag_count*12) + 4)) # bits per pixel
       tiff << short_tag.call( 259, 1, 1 ) # compression
       tiff << short_tag.call( 262, 1, 5 ) # colorspace - separation
-      tiff << long_tag.call( 273, 1, (10 + (tag_count*12) + 16) ) # data offset
+      tiff << long_tag.call( 273, 1, (10 + (tag_count*12) + 20) ) # data offset
       tiff << short_tag.call( 277, 1, 4 ) # samples per pixel
       tiff << long_tag.call( 279, 1, stream.unfiltered_data.size) # data byte size
       tiff << short_tag.call( 284, 1, 1 ) # planer config
       tiff << long_tag.call( 332, 1, 1)   # inkset - CMYK
+      tiff << [0].pack("I") # next IFD pointer
       tiff << [bpc, bpc, bpc, bpc].pack("IIII")
       tiff << stream.unfiltered_data
       File.open(filename, "wb") { |file| file.write tiff }
@@ -119,10 +120,12 @@ module ExtractImages
       tiff << short_tag.call( 258, 1, 8 ) # bits per pixel
       tiff << short_tag.call( 259, 1, 1 ) # compression
       tiff << short_tag.call( 262, 1, 1 ) # colorspace - grayscale
-      tiff << long_tag.call( 273, 1, (10 + (tag_count*12)) ) # data offset
+      tiff << long_tag.call( 273, 1, (10 + (tag_count*12) + 4) ) # data offset
       tiff << short_tag.call( 277, 1, 1 ) # samples per pixel
       tiff << long_tag.call( 279, 1, stream.unfiltered_data.size) # data byte size
       tiff << short_tag.call( 284, 1, 1 ) # planer config
+      tiff << [0].pack("I") # next IFD pointer
+      p stream.unfiltered_data.size
       tiff << stream.unfiltered_data
       File.open(filename, "wb") { |file| file.write tiff }
     end
@@ -144,12 +147,13 @@ module ExtractImages
       tiff = header.dup
       tiff << short_tag.call( 256, 1, w ) # image width
       tiff << short_tag.call( 257, 1, h ) # image height
-      tiff << long_tag.call( 258, 3, (header.size + (tag_count*12))) # bits per pixel
+      tiff << long_tag.call( 258, 3, (header.size + (tag_count*12) + 4)) # bits per pixel
       tiff << short_tag.call( 259, 1, 1 ) # compression
       tiff << short_tag.call( 262, 1, 2 ) # colorspace - RGB
-      tiff << long_tag.call( 273, 1, (header.size + (tag_count*12) + 12) ) # data offset
+      tiff << long_tag.call( 273, 1, (header.size + (tag_count*12) + 16) ) # data offset
       tiff << short_tag.call( 277, 1, 3 ) # samples per pixel
       tiff << long_tag.call( 279, 1, stream.unfiltered_data.size) # data byte size
+      tiff << [0].pack("I") # next IFD pointer
       tiff << [bpc, bpc, bpc].pack("III")
       tiff << stream.unfiltered_data
       File.open(filename, "wb") { |file| file.write tiff }
@@ -209,8 +213,9 @@ module ExtractImages
       + short_tag.call( 256, cols ) \
       + short_tag.call( 257, h ) \
       + short_tag.call( 259, 4 ) \
-      + long_tag.call( 273, (10 + (5*12)) ) \
+      + long_tag.call( 273, (10 + (5*12) + 4) ) \
       + long_tag.call( 279, len) \
+      + [0].pack("I") \
       + stream.data
       File.open(filename, "wb") { |file| file.write tiff }
     end

data/lib/pdf/reader.rb

@@ -25,9 +25,6 @@
 ################################################################################
 
 require 'stringio'
-require 'zlib'
-
-require 'ascii85'
 
 module PDF
   ################################################################################
@@ -113,6 +110,8 @@ module PDF
     #
     def initialize(input = nil, opts = {})
       if input # support the deprecated Reader API
+        @cache   = PDF::Reader::ObjectCache.new
+        opts.merge!(:cache => @cache)
         @objects = PDF::Reader::ObjectHash.new(input, opts)
       end
     end
@@ -222,7 +221,7 @@ module PDF
     #
     def pages
       (1..self.page_count).map { |num|
-        PDF::Reader::Page.new(@objects, num)
+        PDF::Reader::Page.new(@objects, num, :cache => @cache)
       }
     end
 
@@ -241,7 +240,7 @@ module PDF
     def page(num)
       num = num.to_i
       raise ArgumentError, "valid pages are 1 .. #{self.page_count}" if num < 1 || num > self.page_count
-      PDF::Reader::Page.new(@objects, num)
+      PDF::Reader::Page.new(@objects, num, :cache => @cache)
     end
 
 
@@ -338,6 +337,13 @@ require 'pdf/reader/cmap'
 require 'pdf/reader/encoding'
 require 'pdf/reader/error'
 require 'pdf/reader/filter'
+require 'pdf/reader/filter/ascii85'
+require 'pdf/reader/filter/ascii_hex'
+require 'pdf/reader/filter/depredict'
+require 'pdf/reader/filter/flate'
+require 'pdf/reader/filter/lzw'
+require 'pdf/reader/filter/null'
+require 'pdf/reader/filter/run_length'
 require 'pdf/reader/font'
 require 'pdf/reader/form_xobject'
 require 'pdf/reader/glyph_hash'

data/lib/pdf/reader/buffer.rb

@@ -36,7 +36,7 @@ class PDF::Reader
   # the raw tokens into objects we can work with (strings, ints, arrays, etc)
   #
   class Buffer
-    TOKEN_WHITESPACE=["\x00", "\x09", "\x0A", "\x0C", "\x0D", "\x20"]
+    TOKEN_WHITESPACE=[0x00, 0x09, 0x0A, 0x0C, 0x0D, 0x20]
 
     attr_reader :pos
 
@@ -232,18 +232,17 @@ class PDF::Reader
       finished = false
 
       while !finished
-        chr = @io.read(1)
-        codepoint = chr.to_s.unpack("C*").first
-        if chr.nil?
+        byte = @io.getbyte
+        if byte.nil?
           finished = true # unbalanced params
-        elsif (48..57).include?(codepoint) || (65..90).include?(codepoint) || (97..122).include?(codepoint)
-          str << chr
-        elsif codepoint <= 32
+        elsif (48..57).include?(byte) || (65..90).include?(byte) || (97..122).include?(byte)
+          str << byte.chr
+        elsif byte <= 32
           # ignore it
         else
           @tokens << str if str.size > 0
-          @tokens << ">" if chr != ">"
-          @tokens << chr
+          @tokens << ">" if byte != 0x3E # '>'
+          @tokens << byte.chr
           finished = true
         end
       end
@@ -263,19 +262,19 @@ class PDF::Reader
       count = 1
 
       while count > 0
-        chr = @io.read(1)
-        if chr.nil?
+        byte = @io.getbyte
+        if byte.nil?
           count = 0 # unbalanced params
-        elsif chr == "\x5c"
-          str << chr << @io.read(1).to_s
-        elsif chr == "("
+        elsif byte == 0x5C
+          str << byte.chr << @io.getbyte.chr
+        elsif byte == 0x28 # "("
           str << "("
           count += 1
-        elsif chr == ")"
+        elsif byte == 0x29 # ")"
           count -= 1
           str << ")" unless count == 0
         else
-          str << chr unless count == 0
+          str << byte.chr unless count == 0
         end
       end
 
@@ -291,61 +290,68 @@ class PDF::Reader
     def prepare_regular_token
       tok = ""
 
-      while chr = @io.read(1)
-        case chr
-        when "\x25"
+      while byte = @io.getbyte
+        case byte
+        when 0x25
           # comment, ignore everything until the next EOL char
           done = false
           while !done
-            chr = @io.read(1)
-            done = true if chr.nil? || chr == "\x0A" || chr == "\x0D"
+            byte = @io.getbyte
+            done = true if byte.nil? || byte == 0x0A || byte == 0x0D
           end
         when *TOKEN_WHITESPACE
           # white space, token finished
           @tokens << tok if tok.size > 0
 
           #If the token was empty, chomp the rest of the whitespace too
-          while TOKEN_WHITESPACE.include?(peek_char) && tok.size == 0
-            @io.read(1)
+          while TOKEN_WHITESPACE.include?(peek_byte) && tok.size == 0
+            @io.getbyte
           end
           tok = ""
           break
-        when "\x3C"
+        when 0x3C
           # opening delimiter '<', start of new token
           @tokens << tok if tok.size > 0
-          chr << @io.read(1) if peek_char == "\x3C" # check if token is actually '<<'
-          @tokens << chr
+          if peek_byte == 0x3C # check if token is actually '<<'
+            @io.getbyte
+            @tokens << "<<"
+          else
+            @tokens << "<"
+          end
           tok = ""
           break
-        when "\x3E"
+        when 0x3E
           # closing delimiter '>', start of new token
           @tokens << tok if tok.size > 0
-          chr << @io.read(1) if peek_char == "\x3E" # check if token is actually '>>'
-          @tokens << chr
+          if peek_byte == 0x3E # check if token is actually '>>'
+            @io.getbyte
+            @tokens << ">>"
+          else
+            @tokens << byte.chr
+          end
           tok = ""
           break
-        when "\x28", "\x5B", "\x7B"
+        when 0x28, 0x5B, 0x7B
           # opening delimiter, start of new token
           @tokens << tok if tok.size > 0
-          @tokens << chr
+          @tokens << byte.chr
           tok = ""
           break
-        when "\x29", "\x5D", "\x7D"
+        when 0x29, 0x5D, 0x7D
           # closing delimiter
           @tokens << tok if tok.size > 0
-          @tokens << chr
+          @tokens << byte.chr
           tok = ""
           break
-        when "\x2F"
+        when 0x2F
           # PDF name, start of new token
           @tokens << tok if tok.size > 0
-          @tokens << chr
-          next_char = peek_char
-          @tokens << "" if chr == "/" && [nil, " ", "\n"].include?(next_char)
+          @tokens << byte.chr
+          @tokens << "" if byte == 0x2F && [nil, 0x20, 0x0A].include?(peek_byte)
           tok = ""
           break
         else
-          tok << chr
+          tok << byte.chr
         end
       end
 
@@ -355,10 +361,10 @@ class PDF::Reader
     # peek at the next character in the io stream, leaving the stream position
     # untouched
     #
-    def peek_char
-      chr = @io.read(1)
-      @io.seek(-1, IO::SEEK_CUR) unless chr.nil?
-      chr
+    def peek_byte
+      byte = @io.getbyte
+      @io.seek(-1, IO::SEEK_CUR) if byte
+      byte
     end
 
     # for a handful of tokens we want to tell the parser how to convert them

data/lib/pdf/reader/cmap.rb

@@ -60,6 +60,10 @@ class PDF::Reader
       @map.size
     end
 
+    # Convert a glyph code into one or more Codepoints.
+    #
+    # Returns an array of Fixnums.
+    #
     def decode(c)
       # TODO: implement the conversion
       return c unless c.class == Fixnum
@@ -74,12 +78,23 @@ class PDF::Reader
     end
 
     def str_to_int(str)
-      return nil if str.nil? || str.size == 0 || str.size >= 3
-
-      if str.size == 1
-        str.unpack("C*")[0]
+      return nil if str.nil? || str.size == 0
+      unpacked_string = if str.size == 1 # UTF-8
+        str.unpack("C*")
+      else # UTF-16
+         str.unpack("n*")
+      end
+      if unpacked_string.length == 1
+        unpacked_string
+      elsif unpacked_string.length == 2 && (unpacked_string[0] > 0xD800 && unpacked_string[0] < 0xDBFF)
+        # this is a Unicode UTF-16 "Surrogate Pair" see Unicode Spec. Chapter 3.7
+        # lets convert to a UTF-32. (the high bit is between 0xD800-0xDBFF, the
+        # low bit is between 0xDC00-0xDFFF) for example: U+1D44E (U+D835 U+DC4E)
+        [(unpacked_string[0] - 0xD800) * 0x400 + (unpacked_string[1] - 0xDC00) + 0x10000]
       else
-        str.unpack("n*")[0]
+        # it is a bad idea to just return the first 16 bits, as this doesn't allow
+        # for ligatures for example fi (U+0066 U+0069)
+        unpacked_string
       end
     end
 
@@ -88,7 +103,7 @@ class PDF::Reader
       find    = str_to_int(parser.parse_token)
       replace = str_to_int(parser.parse_token)
       while find && replace
-        @map[find] = replace
+        @map[find[0]] = replace
         find       = str_to_int(parser.parse_token)
         replace    = str_to_int(parser.parse_token)
       end
@@ -114,21 +129,21 @@ class PDF::Reader
     end
 
     def bfrange_type_one(start_code, end_code, dst)
-      start_code = str_to_int(start_code)
-      end_code   = str_to_int(end_code)
+      start_code = str_to_int(start_code)[0]
+      end_code   = str_to_int(end_code)[0]
       dst        = str_to_int(dst)
 
       # add all values in the range to our mapping
       (start_code..end_code).each_with_index do |val, idx|
-        @map[val] = dst + idx
+        @map[val] = dst.length == 1 ? [dst[0] + idx] : [dst[0], dst[1] + 1]
         # ensure a single range does not exceed 255 chars
         raise PDF::Reader::MalformedPDFError, "a CMap bfrange cann't exceed 255 chars" if idx > 255
       end
     end
 
     def bfrange_type_two(start_code, end_code, dst)
-      start_code = str_to_int(start_code)
-      end_code   = str_to_int(end_code)
+      start_code = str_to_int(start_code)[0]
+      end_code   = str_to_int(end_code)[0]
       from_range = (start_code..end_code)
 
       # add all values in the range to our mapping

data/lib/pdf/reader/filter.rb

@@ -22,7 +22,6 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 ################################################################################
-require 'zlib'
 
 class PDF::Reader
   ################################################################################
@@ -30,7 +29,7 @@ class PDF::Reader
   # support for features like compression and encryption. This class is for decoding that
   # content.
   #
-  class Filter # :nodoc:
+  module Filter # :nodoc:
 
     ################################################################################
     # creates a new filter for decoding content.
@@ -38,242 +37,20 @@ class PDF::Reader
     # Filters that are only used to encode image data are accepted, but the data is
     # returned untouched. At this stage PDF::Reader has no need to decode images.
     #
-    def initialize (name, options = nil)
-      @options = options
-
+    def self.with(name, options = {})
       case name.to_sym
-      when :ASCII85Decode   then @filter = :ascii85
-      when :ASCIIHexDecode  then @filter = :asciihex
-      when :CCITTFaxDecode  then @filter = nil
-      when :DCTDecode       then @filter = nil
-      when :FlateDecode     then @filter = :flate
-      when :JBIG2Decode     then @filter = nil
-      when :JPXDecode       then @filter = nil
-      when :LZWDecode       then @filter = :lzw
-      when :RunLengthDecode then @filter = :runlength
+      when :ASCII85Decode   then PDF::Reader::Filter::Ascii85.new(options)
+      when :ASCIIHexDecode  then PDF::Reader::Filter::AsciiHex.new(options)
+      when :CCITTFaxDecode  then PDF::Reader::Filter::Null.new(options)
+      when :DCTDecode       then PDF::Reader::Filter::Null.new(options)
+      when :FlateDecode     then PDF::Reader::Filter::Flate.new(options)
+      when :JBIG2Decode     then PDF::Reader::Filter::Null.new(options)
+      when :JPXDecode       then PDF::Reader::Filter::Null.new(options)
+      when :LZWDecode       then PDF::Reader::Filter::Lzw.new(options)
+      when :RunLengthDecode then PDF::Reader::Filter::RunLength.new(options)
       else
         raise UnsupportedFeatureError, "Unknown filter: #{name}"
       end
     end
-    ################################################################################
-    # attempts to decode the specified data with the current filter
-    #
-    # Filters that are only used to encode image data are accepted, but the data is
-    # returned untouched. At this stage PDF::Reader has no need to decode images.
-    #
-    def filter (data)
-      # leave the data untouched if we don't support the required filter
-      return data if @filter.nil?
-
-      # decode the data
-      self.send(@filter, data)
-    end
-    ################################################################################
-    # Decode the specified data using the Ascii85 algorithm. Relies on the AScii85
-    # rubygem.
-    #
-    def ascii85(data)
-      data = "<~#{data}" unless data.to_s[0,2] == "<~"
-      Ascii85::decode(data)
-    rescue Exception => e
-      # Oops, there was a problem decoding the stream
-      raise MalformedPDFError, "Error occured while decoding an ASCII85 stream (#{e.class.to_s}: #{e.to_s})"
-    end
-    ################################################################################
-    # Decode the specified data using the AsciiHex algorithm.
-    #
-    def asciihex(data)
-      data.chop! if data[-1,1] == ">"
-      data = data[1,data.size] if data[0,1] == "<"
-      data.gsub!(/[^A-Fa-f0-9]/,"")
-      data << "0" if data.size % 2 == 1
-      data.scan(/.{2}/).map { |s| s.hex.chr }.join("")
-    rescue Exception => e
-      # Oops, there was a problem decoding the stream
-      raise MalformedPDFError, "Error occured while decoding an ASCIIHex stream (#{e.class.to_s}: #{e.to_s})"
-    end
-    ################################################################################
-    # Decode the specified data with the Zlib compression algorithm
-    def flate (data)
-      deflated = nil
-      begin
-        deflated = Zlib::Inflate.new.inflate(data)
-      rescue Zlib::DataError => e
-        # by default, Ruby's Zlib assumes the data it's inflating
-        # is RFC1951 deflated data, wrapped in a RFC1951 zlib container.
-        # If that fails, then use an undocumented 'feature' to attempt to inflate
-        # the data as a raw RFC1951 stream.
-        #
-        # See
-        # - http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/243545
-        # - http://www.gzip.org/zlib/zlib_faq.html#faq38
-        deflated = Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(data)
-      end
-      depredict(deflated, @options)
-    rescue Exception => e
-      # Oops, there was a problem inflating the stream
-      raise MalformedPDFError, "Error occured while inflating a compressed stream (#{e.class.to_s}: #{e.to_s})"
-    end
-    ################################################################################
-    # Decode the specified data with the LZW compression algorithm
-    def lzw(data)
-      data = PDF::Reader::LZW.decode(data)
-      depredict(data, @options)
-    end
-    ################################################################################
-    # Decode the specified data with the RunLengthDecode compression algorithm
-    def runlength(data)
-      pos = 0
-      out = ""
-
-      while pos < data.length
-        if data.respond_to?(:getbyte)
-          length = data.getbyte(pos)
-        else
-          length = data[pos]
-        end
-        pos += 1
-
-        case
-        when length == 128
-          break
-        when length < 128
-          # When the length is < 128, we copy the following length+1 bytes
-          # literally.
-          out << data[pos, length + 1]
-          pos += length
-        else
-          # When the length is > 128, we copy the next byte (257 - length)
-          # times; i.e., "\xFA\x00" ([250, 0]) will expand to
-          # "\x00\x00\x00\x00\x00\x00\x00".
-          out << data[pos, 1] * (257 - length)
-        end
-
-        pos += 1
-      end
-
-      out
-    end
-    ################################################################################
-    def depredict(data, opts = {})
-      predictor = (opts || {})[:Predictor].to_i
-
-      case predictor
-      when 0, 1 then
-        data
-      when 2    then
-        tiff_depredict(data, opts)
-      when 10, 11, 12, 13, 14, 15 then
-        png_depredict(data, opts)
-      else
-        raise  MalformedPDFError, "Unrecognised predictor value (#{predictor})"
-      end
-    end
-    ################################################################################
-    def tiff_depredict(data, opts = {})
-      data        = data.unpack("C*")
-      unfiltered  = []
-      bpc         = opts[:BitsPerComponent] || 8
-      pixel_bits  = bpc * opts[:Colors]
-      pixel_bytes = pixel_bits / 8
-      line_len    = (pixel_bytes * opts[:Columns])
-      pos         = 0
-
-      if bpc != 8
-        raise UnsupportedFeatureError, "TIFF predictor onlys supports 8 Bits Per Component"
-      end
-
-      until pos > data.size
-        row_data = data[pos, line_len]
-        row_data.each_with_index do |byte, index|
-          left = index < pixel_bytes ? 0 : row_data[index - pixel_bytes]
-          row_data[index] = (byte + left) % 256
-        end
-        unfiltered += row_data
-        pos += line_len
-      end
-
-      unfiltered.pack("C*")
-    end
-    ################################################################################
-    def png_depredict(data, opts = {})
-      return data if opts.nil? || opts[:Predictor].to_i < 10
-
-      data = data.unpack("C*")
-
-      pixel_bytes     = opts[:Colors] || 1
-      scanline_length = (pixel_bytes * opts[:Columns]) + 1
-      row = 0
-      pixels = []
-      paeth, pa, pb, pc = nil
-      until data.empty? do
-        row_data = data.slice! 0, scanline_length
-        filter = row_data.shift
-        case filter
-        when 0 # None
-        when 1 # Sub
-          row_data.each_with_index do |byte, index|
-            left = index < pixel_bytes ? 0 : row_data[index - pixel_bytes]
-            row_data[index] = (byte + left) % 256
-            #p [byte, left, row_data[index]]
-          end
-        when 2 # Up
-          row_data.each_with_index do |byte, index|
-            col = index / pixel_bytes
-            upper = row == 0 ? 0 : pixels[row-1][col][index % pixel_bytes]
-            row_data[index] = (upper + byte) % 256
-          end
-        when 3  # Average
-          row_data.each_with_index do |byte, index|
-            col = index / pixel_bytes
-            upper = row == 0 ? 0 : pixels[row-1][col][index % pixel_bytes]
-            left = index < pixel_bytes ? 0 : row_data[index - pixel_bytes]
-
-            row_data[index] = (byte + ((left + upper)/2).floor) % 256
-          end
-        when 4 # Paeth
-          left = upper = upper_left = nil
-          row_data.each_with_index do |byte, index|
-            col = index / pixel_bytes
-
-            left = index < pixel_bytes ? 0 : row_data[index - pixel_bytes]
-            if row.zero?
-              upper = upper_left = 0
-            else
-              upper = pixels[row-1][col][index % pixel_bytes]
-              upper_left = col.zero? ? 0 :
-                pixels[row-1][col-1][index % pixel_bytes]
-            end
-
-            p = left + upper - upper_left
-            pa = (p - left).abs
-            pb = (p - upper).abs
-            pc = (p - upper_left).abs
-
-            paeth = if pa <= pb && pa <= pc
-                      left
-                    elsif pb <= pc
-                      upper
-                    else
-                      upper_left
-                    end
-
-            row_data[index] = (byte + paeth) % 256
-          end
-        else
-          raise ArgumentError, "Invalid filter algorithm #{filter}"
-        end
-
-        s = []
-        row_data.each_slice pixel_bytes do |slice|
-          s << slice
-        end
-        pixels << s
-        row += 1
-      end
-
-      pixels.map { |bytes| bytes.flatten.pack("C*") }.join("")
-    end
   end
 end
-################################################################################