substrate 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20af64cb9a41e61ad9af8a5f678f3485947689193ad3f71d7638bd332452c88b
-  data.tar.gz: 34bc75ca3736801c7f35d968a626e32a597d62c36ef6516b7f727c1cd21e53ff
+  metadata.gz: f70ea3ee7d06a49e41215d92a1b04db9173cbf357cbeccbd9bb85adc281b2e45
+  data.tar.gz: b86bcb7ff7b2f9eab771a2e3419b1ab0cf42a08bb4ae6d4bd906d2cee0ea0c5f
 SHA512:
-  metadata.gz: 46c461e08228dcf87eed7d6e6c791619b3a83fc6ee72f1e2804db5e9d81a606d0299afdd3a69c5ce4ba52dc3a3fff1720db1dee30ad1cb810fb7f1bcbcc855d6
-  data.tar.gz: 744c7900ef351097160f26a695c440f1197b07ad9aafd38c06d9ce86a5a6714d445d766d048c1cd8ed69cabef792618e4879b19b5383d599441908d3d59c6447
+  metadata.gz: 2d2687f91dbe8ca128de93e975de5e0a25359e8e8f1d7399b6092721b553ac26985eeff2d2963a7112f70bf11f98fdbcfff8f30e4462cb9fc9b3375349e6cf37
+  data.tar.gz: e303be1550bd6d5f98635689316e9be8e17d88657e196d614133f84aad0674745db45ffb0f1702dfc22ac3f5798e923113f612151fe98a71bfbce8ba602b0542

data/lib/substrate/extract_pdf.rb

@@ -5,14 +5,16 @@ require "stringio"
 
 module Substrate
   class ExtractPdf < Command
-    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new)
+    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new, page_separator: "\n\n")
       @bytes = bytes
       @source_ref = source_ref
       @limits = limits
+      @page_separator = page_separator
     end
 
     def call
       validate_input
+      validate_page_separator
       extract_document
     end
 
@@ -23,6 +25,14 @@ module Substrate
       @limits.check!(:input_bytes, @bytes.bytesize)
     end
 
+    def validate_page_separator
+      raise ArgumentError, "page_separator must be a String" unless @page_separator.is_a?(String)
+      @page_separator = @page_separator.encode(Encoding::UTF_8).dup.freeze
+      raise ArgumentError, "page_separator must contain valid UTF-8" unless @page_separator.valid_encoding?
+    rescue EncodingError => error
+      raise ArgumentError, "page_separator must contain valid UTF-8", cause: error
+    end
+
     def extract_document
       StringIO.open(@bytes) do |io|
         document = HexaPDF::Document.new(io: io, config: {"document.auto_decrypt" => false, "filter.map" => {BrotliDecode: nil}})
@@ -69,14 +79,13 @@ module Substrate
     end
 
     def extract_pages(objects)
-      text_bytes = 0
+      @text = +""
       objects.map.with_index do |page, index|
         @limits.check!(:pages, index + 1)
         text = page.extract_text.encode(Encoding::UTF_8)
         raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8 or NUL" unless text.valid_encoding? && !text.include?("\0")
-        text_bytes += text.bytesize + ((index == 0) ? 0 : 2)
-        @limits.check!(:text_bytes, text_bytes)
-        Mediums::Pdf::Page.new(number: index + 1, text: text.freeze)
+        append_page_text(text, index)
+        Mediums::Pdf::Page.new(number: index + 1, text: text.freeze, crop_box: crop_box(page))
       end.freeze.tap do |pages|
         raise Mediums::Pdf::NoSelectableText, "PDF contains no selectable text" if pages.all? { |page| page.text.match?(/\A[[:space:]]*\z/) }
       end
@@ -84,12 +93,41 @@ module Substrate
       raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8", cause: error
     end
 
+    def append_page_text(text, index)
+      separator = index.zero? ? "" : @page_separator
+      @limits.check!(:text_bytes, @text.bytesize + separator.bytesize + text.bytesize)
+      @text << separator << text
+    end
+
+    def crop_box(page)
+      media, crop = page_rectangles(page)
+      return unless usable_box?(media) && usable_box?(crop)
+      box = Mediums::Pdf::Box.new(left: [media.left, crop.left].max,
+        bottom: [media.bottom, crop.bottom].max, right: [media.right, crop.right].min,
+        top: [media.top, crop.top].min)
+      box if box.right > box.left && box.top > box.bottom
+    end
+
+    def page_rectangles(page)
+      media = page[:MediaBox]
+      crop = page[:CropBox]
+      [media, crop.nil? ? media : crop]
+    rescue HexaPDF::Error
+      [nil, nil]
+    end
+
+    def usable_box?(rectangle)
+      rectangle.is_a?(HexaPDF::Rectangle) &&
+        rectangle.to_a.all? { |value| value.is_a?(Numeric) && value.finite? } &&
+        rectangle.right > rectangle.left && rectangle.top > rectangle.bottom
+    end
+
     def publish_content(pages, outline)
       metadata = {extractor: "HexaPDF", extractor_version: HexaPDF::VERSION.dup.freeze,
                   input_bytes: @bytes.bytesize, page_count: pages.size,
-                  text_bytes: pages.sum { |page| page.text.bytesize } + 2 * (pages.size - 1)}.freeze
+                  text_bytes: @text.bytesize}.freeze
       success(payload: Mediums::Pdf::Content.new(pages: pages, source_ref: @source_ref,
-        metadata: metadata, outline: outline))
+        metadata: metadata, outline: outline, text: @text.freeze))
     end
   end
 end

data/lib/substrate/mediums/pdf/box.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Box = Data.define(:left, :bottom, :right, :top)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/content.rb

@@ -3,11 +3,7 @@
 module Substrate
   module Mediums
     module Pdf
-      Content = Data.define(:pages, :source_ref, :metadata, :outline) do
-        def text
-          pages.map(&:text).join("\n\n").freeze
-        end
-      end
+      Content = Data.define(:pages, :source_ref, :metadata, :outline, :text)
     end
   end
 end

data/lib/substrate/mediums/pdf/page.rb

@@ -3,7 +3,7 @@
 module Substrate
   module Mediums
     module Pdf
-      Page = Data.define(:number, :text)
+      Page = Data.define(:number, :text, :crop_box)
     end
   end
 end

data/lib/substrate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Substrate
-  VERSION = "0.2.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: substrate
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Dan Frenette
@@ -98,6 +98,7 @@ files:
 - lib/substrate/handlers/raw_text.rb
 - lib/substrate/handlers/text_file.rb
 - lib/substrate/handlers/transcript.rb
+- lib/substrate/mediums/pdf/box.rb
 - lib/substrate/mediums/pdf/chapter.rb
 - lib/substrate/mediums/pdf/content.rb
 - lib/substrate/mediums/pdf/destination.rb