pdf-reader 2.7.0 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5ee0d8c3c55f6a0aebb60a0a6dce92428e8371b96a6beb6d75bfe90602bffae7
-  data.tar.gz: '0911d108353bf577aa9fd7b49b97dda1cf9d54816bf8ff6c4225281eeda63229'
+  metadata.gz: 6182ffd59631afba6a2c234547a428382b1ec2d7b414d89830b1143f1a0e1704
+  data.tar.gz: 6c0e6a7d32cf24912edc3aa96d72b7f70497d2fdd0e0913b86f871bbf9fa104f
 SHA512:
-  metadata.gz: 917db2b1fb977b41e7b057ff3d215b8f249577254d9fe3df72f330b32ff49630874c58f480495ddcd137d9f31d014083438623cdf7260b0d7a87bbe3a5f3685a
-  data.tar.gz: cd9832f025264e54d586e81eff69727379e8646d741f53ae61e90a5b38945d852147853891d468bab683581bdd0beb68a9b7c7f5e54e064e9a3935262ea9d651
+  metadata.gz: 42dafbe0c36ce838da4c3120bf2187efde647e486971896d9a9c59c37dac3da0f2ccf3ecd98d8dd1d3acc5404bfcf26e64a327d7797648646afd6b40be02fec2
+  data.tar.gz: 40f0b0958024b558d6aca7eb2b3b6f042f034059c8fca52ce97fab7d55a39c313797605341331c65efd1099a1310ccbe386c354024dbd3cbc61c1d96c423842d

data/CHANGELOG

@@ -1,3 +1,11 @@
+v2.8.0 (28th Decemeber 2021)
+- Add PDF::Reader::Page#runs for extracting text from a page with positioning metadata (http://github.com/yob/pdf-reader/pull/411)
+- Add options to PDF::Reader::Page#text to make some behaviour configurable (http://github.com/yob/pdf-reader/pull/411)
+    - including extracting the text for only part of the page
+- Improve text positioning and extraction for Type3 fonts (http://github.com/yob/pdf-reader/pull/412)
+- Skip extracting text that is positioned outside the page (http://github.com/yob/pdf-reader/pull/413)
+- Fix occasional crash when reading some streams (http://github.com/yob/pdf-reader/pull/405)
+
 v2.7.0 (13th December 2021)
 - Include RBI type files in the gem
   - Downstream users of pdf-reader who also use sorbet *should* find many parts of the API will

data/Rakefile

@@ -14,7 +14,7 @@ desc "Run cane to check quality metrics"
 Cane::RakeTask.new(:quality) do |cane|
   cane.abc_max = 20
   cane.style_measure = 100
-  cane.max_violations = 32
+  cane.max_violations = 28
 
   cane.use Morecane::EncodingCheck, :encoding_glob => "{app,lib,spec}/**/*.rb"
 end

data/lib/pdf/reader/bounding_rectangle_runs_filter.rb

@@ -0,0 +1,16 @@
+# coding: utf-8
+# typed: strict
+# frozen_string_literal: true
+
+class PDF::Reader
+
+  # Filter our text/characters that are positioned outside a rectangle. Usually the page
+  # MediaBox or CropBox, but could be a user specified rectangle too
+  class BoundingRectangleRunsFilter
+
+    def self.runs_within_rect(runs, rect)
+      runs.select { |run| rect.contains?(run.origin) }
+    end
+  end
+end
+

data/lib/pdf/reader/font.rb

@@ -43,6 +43,7 @@ class PDF::Reader
       @tounicode = nil
 
       extract_base_info(obj)
+      extract_type3_info(obj)
       extract_descriptor(obj)
       extract_descendants(obj)
       @width_calc = build_width_calculator
@@ -73,8 +74,44 @@ class PDF::Reader
       @cached_widths[code_point] ||= @width_calc.glyph_width(code_point)
     end
 
+    # In most cases glyph width is converted into text space with a simple divide by 1000.
+    #
+    # However, Type3 fonts provide their own FontMatrix that's used for the transformation.
+    #
+    def glyph_width_in_text_space(code_point)
+      glyph_width_in_glyph_space = glyph_width(code_point)
+
+      if @subtype == :Type3
+        x1, y1 = font_matrix_transform(0,0)
+        x2, y2 = font_matrix_transform(glyph_width_in_glyph_space, 0)
+        (x2 - x1).abs.round(2)
+      else
+        glyph_width_in_glyph_space / 1000.0
+      end
+    end
+
     private
 
+    # Only valid for Type3 fonts
+    def font_matrix_transform(x, y)
+      return x, y if @font_matrix.nil?
+
+      matrix = TransformationMatrix.new(
+        @font_matrix[0], @font_matrix[1],
+        @font_matrix[2], @font_matrix[3],
+        @font_matrix[4], @font_matrix[5],
+      )
+
+      if x == 0 && y == 0
+        [matrix.e, matrix.f]
+      else
+        [
+          (matrix.a * x) + (matrix.c * y) + (matrix.e),
+          (matrix.b * x) + (matrix.d * y) + (matrix.f)
+        ]
+      end
+    end
+
     def default_encoding(font_name)
       case font_name.to_s
       when "Symbol" then
@@ -138,6 +175,12 @@ class PDF::Reader
       end
     end
 
+    def extract_type3_info(obj)
+      if @subtype == :Type3
+        @font_matrix = @ohash.object(obj[:FontMatrix]) || [ 0.001, 0, 0, 0.001, 0, 0 ]
+      end
+    end
+
     def extract_descriptor(obj)
       if obj[:FontDescriptor]
         # create a font descriptor object if we can, in other words, unless this is

data/lib/pdf/reader/page.rb

@@ -101,13 +101,24 @@ module PDF
       # returns the plain text content of this page encoded as UTF-8. Any
       # characters that can't be translated will be returned as a ▯
       #
-      def text
+      def text(opts = {})
         receiver = PageTextReceiver.new
         walk(receiver)
-        receiver.content
+        runs = receiver.runs(opts)
+
+        # rectangles[:MediaBox] can never be nil, but I have no easy way to tell sorbet that atm
+        mediabox = rectangles[:MediaBox] || Rectangle.new(0, 0, 0, 0)
+
+        PageLayout.new(runs, mediabox).to_s
       end
       alias :to_s :text
 
+      def runs(opts = {})
+        receiver = PageTextReceiver.new
+        walk(receiver)
+        receiver.runs(opts)
+      end
+
       # processes the raw content stream for this page in sequential order and
       # passes callbacks to the receiver objects.
       #

data/lib/pdf/reader/page_layout.rb

@@ -21,10 +21,8 @@ class PDF::Reader
       # PDF::Reader::Rectangle at some point
       PDF::Reader::Error.validate_not_nil(mediabox, "mediabox")
 
-      runs = ZeroWidthRunsFilter.exclude_zero_width_runs(runs)
-      runs = OverlappingRunsFilter.exclude_redundant_runs(runs)
-      @mediabox = mediabox
-      @runs = merge_runs(runs)
+      @mediabox = process_mediabox(mediabox)
+      @runs = runs
       @mean_font_size   = mean(@runs.map(&:font_size)) || DEFAULT_FONT_SIZE
       @mean_font_size = DEFAULT_FONT_SIZE if @mean_font_size == 0
       @median_glyph_width = median(@runs.map(&:mean_character_width)) || 0
@@ -51,13 +49,11 @@ class PDF::Reader
     private
 
     def page_width
-      # TODO once @mediabox is a Rectangle, this can be just `@mediabox.width`
-      (@mediabox[2].to_f - @mediabox[0].to_f).abs
+      @mediabox.width
     end
 
     def page_height
-      # TODO once @mediabox is a Rectangle, this can be just `@mediabox.height`
-      (@mediabox[3].to_f - @mediabox[1].to_f).abs
+      @mediabox.height
     end
 
     # given an array of strings, return a new array with empty rows from the
@@ -109,30 +105,20 @@ class PDF::Reader
       end
     end
 
-    # take a collection of TextRun objects and merge any that are in close
-    # proximity
-    def merge_runs(runs)
-      runs.group_by { |char|
-        char.y.to_i
-      }.map { |y, chars|
-        group_chars_into_runs(chars.sort)
-      }.flatten.sort
+    def local_string_insert(haystack, needle, index)
+      haystack[Range.new(index, index + needle.length - 1)] = String.new(needle)
     end
 
-    def group_chars_into_runs(chars)
-      chars.each_with_object([]) do |char, runs|
-        if runs.empty?
-          runs << char
-        elsif runs.last.mergable?(char)
-          runs[-1] = runs.last + char
-        else
-          runs << char
-        end
+    def process_mediabox(mediabox)
+      if mediabox.is_a?(Array)
+        msg = "Passing the mediabox to PageLayout as an Array is deprecated," +
+          " please use a Rectangle instead"
+        $stderr.puts msg
+        PDF::Reader::Rectangle.from_array(mediabox)
+      else
+        mediabox
       end
     end
 
-    def local_string_insert(haystack, needle, index)
-      haystack[Range.new(index, index + needle.length - 1)] = String.new(needle)
-    end
   end
 end

data/lib/pdf/reader/page_text_receiver.rb

@@ -47,9 +47,32 @@ module PDF
         @characters = []
       end
 
+      def runs(opts = {})
+        runs = @characters
+
+        if rect = opts.fetch(:rect, @page.rectangles[:CropBox])
+          runs = BoundingRectangleRunsFilter.runs_within_rect(runs, rect)
+        end
+
+        if opts.fetch(:skip_zero_width, true)
+          runs = ZeroWidthRunsFilter.exclude_zero_width_runs(runs)
+        end
+
+        if opts.fetch(:skip_overlapping, true)
+          runs = OverlappingRunsFilter.exclude_redundant_runs(runs)
+        end
+
+        if opts.fetch(:merge, true)
+          runs = merge_runs(runs)
+        end
+
+        runs
+      end
+
+      # deprecated
       def content
-        mediabox = @page.rectangles[:MediaBox].to_a
-        PageLayout.new(@characters, mediabox).to_s
+        mediabox = @page.rectangles[:MediaBox]
+        PageLayout.new(runs, mediabox).to_s
       end
 
       #####################################################
@@ -109,7 +132,7 @@ module PDF
 
           # apply to glyph displacment for the current glyph so the next
           # glyph will appear in the correct position
-          glyph_width = @state.current_font.glyph_width(glyph_code) / 1000.0
+          glyph_width = @state.current_font.glyph_width_in_text_space(glyph_code)
           th = 1
           scaled_glyph_width = glyph_width * @state.font_size * th
           unless utf8_chars == SPACE
@@ -119,12 +142,6 @@ module PDF
         end
       end
 
-      # TODO: revist this. It rotates the co-ordinates to the right direction, but I don't
-      #       think it sets the correct x,y values. We get away with it because we don't
-      #       return the text with co-ordinates, only the full text arranged in a string.
-      #
-      #       We should provide an API for extracting the text with positioning data and spec
-      #       that. I suspect the co-ords might be wrong for rotated pages
       def apply_rotation(x, y)
         if @page.rotate == 90
           tmp = x
@@ -141,6 +158,28 @@ module PDF
         return x, y
       end
 
+      # take a collection of TextRun objects and merge any that are in close
+      # proximity
+      def merge_runs(runs)
+        runs.group_by { |char|
+          char.y.to_i
+        }.map { |y, chars|
+          group_chars_into_runs(chars.sort)
+        }.flatten.sort
+      end
+
+      def group_chars_into_runs(chars)
+        chars.each_with_object([]) do |char, runs|
+          if runs.empty?
+            runs << char
+          elsif runs.last.mergable?(char)
+            runs[-1] = runs.last + char
+          else
+            runs << char
+          end
+        end
+      end
+
     end
   end
 end

data/lib/pdf/reader/parser.rb

@@ -210,6 +210,9 @@ class PDF::Reader
       raise MalformedPDFError, "PDF malformed, missing stream length" unless dict.has_key?(:Length)
       if @objects
         length = @objects.deref(dict[:Length])
+        if dict[:Filter]
+          dict[:Filter] = @objects.deref(dict[:Filter])
+        end
       else
         length = dict[:Length] || 0
       end

data/lib/pdf/reader/rectangle.rb

@@ -26,6 +26,19 @@ module PDF
         set_corners(x1, y1, x2, y2)
       end
 
+      def self.from_array(arr)
+        if arr.size != 4
+          raise ArgumentError, "Only 4-element Arrays can be converted to a Rectangle"
+        end
+
+        PDF::Reader::Rectangle.new(
+          arr[0].to_f,
+          arr[1].to_f,
+          arr[2].to_f,
+          arr[3].to_f,
+        )
+      end
+
       def ==(other)
         to_a == other.to_a
       end
@@ -38,6 +51,11 @@ module PDF
         bottom_right.x - bottom_left.x
       end
 
+      def contains?(point)
+        point.x >= bottom_left.x && point.x <= top_right.x &&
+          point.y >= bottom_left.y && point.y <= top_right.y
+      end
+
       # A pdf-style 4-number array
       def to_a
         [

data/lib/pdf/reader/text_run.rb

@@ -7,15 +7,14 @@ class PDF::Reader
   class TextRun
     include Comparable
 
-    attr_reader :x, :y, :width, :font_size, :text
+    attr_reader :origin, :width, :font_size, :text
 
     alias :to_s :text
 
     def initialize(x, y, width, font_size, text)
-      @x = x
-      @y = y
+      @origin = PDF::Reader::Point.new(x, y)
       @width = width
-      @font_size = font_size.floor
+      @font_size = font_size
       @text = text
     end
 
@@ -35,12 +34,20 @@ class PDF::Reader
       end
     end
 
+    def x
+      @origin.x
+    end
+
+    def y
+      @origin.y
+    end
+
     def endx
-      @endx ||= x + width
+      @endx ||= @origin.x + width
     end
 
     def endy
-      @endy ||= y + font_size
+      @endy ||= @origin.y + font_size
     end
 
     def mean_character_width

data/lib/pdf/reader.rb

@@ -112,17 +112,25 @@ module PDF
     #
     #   reader = PDF::Reader.new("somefile.pdf", :password => "apples")
     #
+    # Using this method directly is supported, but it's more common to use
+    # `PDF::Reader.open`
+    #
     def initialize(input, opts = {})
       @cache   = PDF::Reader::ObjectCache.new
       opts.merge!(:cache => @cache)
       @objects = PDF::Reader::ObjectHash.new(input, opts)
     end
 
+    # Return a Hash with some basic information about the PDF file
+    #
     def info
       dict = @objects.deref(@objects.trailer[:Info])
       doc_strings_to_utf8(dict)
     end
 
+    # Return a Hash with extra metadata provided by the author of the PDF file. Not
+    # always present.
+    #
     def metadata
       stream = @objects.deref(root[:Metadata])
       if stream.nil?
@@ -134,6 +142,8 @@ module PDF
       end
     end
 
+    # To number of pages in this PDF
+    #
     def page_count
       pages = @objects.deref(root[:Pages])
       unless pages.kind_of?(::Hash)
@@ -142,12 +152,14 @@ module PDF
       @page_count ||= @objects.deref(pages[:Count])
     end
 
+    # The PDF version this file uses
+    #
     def pdf_version
       @objects.pdf_version
     end
 
-    # syntactic sugar for opening a PDF file. Accepts the same arguments
-    # as new().
+    # syntactic sugar for opening a PDF file and the most common approach. Accepts the
+    # same arguments as new().
     #
     #   PDF::Reader.open("somefile.pdf") do |reader|
     #     puts reader.pdf_version
@@ -273,6 +285,7 @@ end
 
 require 'pdf/reader/resource_methods'
 require 'pdf/reader/buffer'
+require 'pdf/reader/bounding_rectangle_runs_filter'
 require 'pdf/reader/cid_widths'
 require 'pdf/reader/cmap'
 require 'pdf/reader/encoding'

data/rbi/pdf-reader.rbi

@@ -43,6 +43,13 @@ module PDF
     sig { returns(T::Hash[Symbol, T.untyped]) }
     def root; end
 
+    class BoundingRectangleRunsFilter
+      extend T::Sig
+
+      sig { params(runs: T::Array[PDF::Reader::TextRun], rect: PDF::Reader::Rectangle).returns(T::Array[PDF::Reader::TextRun]) }
+      def self.runs_within_rect(runs, rect); end
+    end
+
     class Buffer
       TOKEN_WHITESPACE = [0x00, 0x09, 0x0A, 0x0C, 0x0D, 0x20]
       TOKEN_DELIMITER = [0x25, 0x3C, 0x3E, 0x28, 0x5B, 0x7B, 0x29, 0x5D, 0x7D, 0x2F]
@@ -750,8 +757,11 @@ module PDF
       sig { returns(T::Array[Numeric]) }
       def origin; end
 
-      sig { returns(String) }
-      def text; end
+      sig { params(opts: T::Hash[Symbol, T.untyped]).returns(T::Array[PDF::Reader::TextRun]) }
+      def runs(opts = {}); end
+
+      sig { params(opts: T::Hash[Symbol, T.untyped]).returns(String) }
+      def text(opts = {}); end
 
       sig { params(receivers: T.untyped).void }
       def walk(*receivers); end
@@ -794,7 +804,7 @@ module PDF
       extend T::Sig
       DEFAULT_FONT_SIZE = 12
 
-      sig { params(runs: T::Array[PDF::Reader::TextRun], mediabox: T::Array[Numeric]).void }
+      sig { params(runs: T::Array[PDF::Reader::TextRun], mediabox: T.any(T::Array[Numeric], PDF::Reader::Rectangle)).void }
       def initialize(runs, mediabox); end
 
       sig { returns(String) }
@@ -829,6 +839,9 @@ module PDF
 
       sig { params(haystack: T.untyped, needle: T.untyped, index: T.untyped).returns(T.untyped) }
       def local_string_insert(haystack, needle, index); end
+
+      sig { params(mediabox: T.untyped).returns(T.untyped) }
+      def process_mediabox(mediabox); end
     end
 
     class PageState
@@ -996,6 +1009,9 @@ module PDF
       sig { params(str: T.untyped).returns(T.untyped) }
       def move_to_next_line_and_show_text(str); end
 
+      sig { params(opts: T::Hash[Symbol, T.untyped]).returns(T::Array[PDF::Reader::TextRun]) }
+      def runs(opts = {}); end
+
       sig { params(aw: T.untyped, ac: T.untyped, string: T.untyped).returns(T.untyped) }
       def set_spacing_next_line_show_text(aw, ac, string); end
 
@@ -1122,6 +1138,9 @@ module PDF
     end
 
     class Rectangle
+      sig { params(arr: T::Array[Numeric]).returns(PDF::Reader::Rectangle) }
+      def self.from_array(arr); end
+
       sig do
         params(
           x1: Numeric,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pdf-reader
 version: !ruby/object:Gem::Version
-  version: 2.7.0
+  version: 2.8.0
 platform: ruby
 authors:
 - James Healy
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-13 00:00:00.000000000 Z
+date: 2021-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -215,6 +215,7 @@ files:
 - lib/pdf/reader/afm/Times-Italic.afm
 - lib/pdf/reader/afm/Times-Roman.afm
 - lib/pdf/reader/afm/ZapfDingbats.afm
+- lib/pdf/reader/bounding_rectangle_runs_filter.rb
 - lib/pdf/reader/buffer.rb
 - lib/pdf/reader/cid_widths.rb
 - lib/pdf/reader/cmap.rb
@@ -281,9 +282,9 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/yob/pdf-reader/issues
-  changelog_uri: https://github.com/yob/pdf-reader/blob/v2.7.0/CHANGELOG
-  documentation_uri: https://www.rubydoc.info/gems/pdf-reader/2.7.0
-  source_code_uri: https://github.com/yob/pdf-reader/tree/v2.7.0
+  changelog_uri: https://github.com/yob/pdf-reader/blob/v2.8.0/CHANGELOG
+  documentation_uri: https://www.rubydoc.info/gems/pdf-reader/2.8.0
+  source_code_uri: https://github.com/yob/pdf-reader/tree/v2.8.0
 post_install_message:
 rdoc_options:
 - "--title"