substrate 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f70ea3ee7d06a49e41215d92a1b04db9173cbf357cbeccbd9bb85adc281b2e45
-  data.tar.gz: b86bcb7ff7b2f9eab771a2e3419b1ab0cf42a08bb4ae6d4bd906d2cee0ea0c5f
+  metadata.gz: 58f4512e0069be50f46373801d5404c6f0a686d05ae4d1cbb605dab1a301cdeb
+  data.tar.gz: 37a9327014aa90f83a81dc57d99a789afb0929e1bfc559c685a7e2e69baff610
 SHA512:
-  metadata.gz: 2d2687f91dbe8ca128de93e975de5e0a25359e8e8f1d7399b6092721b553ac26985eeff2d2963a7112f70bf11f98fdbcfff8f30e4462cb9fc9b3375349e6cf37
-  data.tar.gz: e303be1550bd6d5f98635689316e9be8e17d88657e196d614133f84aad0674745db45ffb0f1702dfc22ac3f5798e923113f612151fe98a71bfbce8ba602b0542
+  metadata.gz: e5774d27284b16021e0d61b822f60dabc6bce5cf4de7bd38bbd4b3e606549137a75a0c45bde340628746a114666bca50a758c101ab51ad816728df1469038bd2
+  data.tar.gz: c1d796ae557619eb52da2d57838e9f84fcf3a8cd57aaa62ada4a9fb35fcbc7ca736fea662da6b9d42c79adfd986a5381ee5145de8c87455306a69e96baae7e44

data/lib/substrate/extract_pdf.rb

@@ -5,7 +5,7 @@ require "stringio"
 
 module Substrate
   class ExtractPdf < Command
-    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new, page_separator: "\n\n")
+    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new, page_separator: Mediums::Pdf::Content::DEFAULT_PAGE_SEPARATOR)
       @bytes = bytes
       @source_ref = source_ref
       @limits = limits
@@ -125,7 +125,7 @@ module Substrate
     def publish_content(pages, outline)
       metadata = {extractor: "HexaPDF", extractor_version: HexaPDF::VERSION.dup.freeze,
                   input_bytes: @bytes.bytesize, page_count: pages.size,
-                  text_bytes: @text.bytesize}.freeze
+                  text_bytes: @text.bytesize, page_separator: @page_separator}.freeze
       success(payload: Mediums::Pdf::Content.new(pages: pages, source_ref: @source_ref,
         metadata: metadata, outline: outline, text: @text.freeze))
     end

data/lib/substrate/mediums/pdf/content.rb

@@ -3,7 +3,13 @@
 module Substrate
   module Mediums
     module Pdf
-      Content = Data.define(:pages, :source_ref, :metadata, :outline, :text)
+      class Content < Data.define(:pages, :source_ref, :metadata, :outline, :text)
+        DEFAULT_PAGE_SEPARATOR = "\n\n"
+
+        def page_separator
+          metadata.fetch(:page_separator, DEFAULT_PAGE_SEPARATOR)
+        end
+      end
     end
   end
 end

data/lib/substrate/mediums/pdf/heading_index.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class HeadingIndex
+        MAX_HEADING_LINES = 3
+
+        def initialize(pages:, entries:)
+          @pages = pages
+          @entries = entries
+          @headings_by_page = {}
+        end
+
+        def find(page, title)
+          headings_by_page[page] ||= index_page(page)
+          headings_by_page.fetch(page)[normalize(title)]
+        end
+
+        private
+
+        attr_reader :entries, :headings_by_page, :pages
+
+        def titles_by_page
+          @titles_by_page ||= entries.group_by { |entry| entry.destination.page_number }
+            .transform_values { |page_entries| page_entries.to_h { |entry| [normalize(entry.title), nil] } }
+        end
+
+        def index_page(page)
+          recent_lines = []
+          offset = 0
+          pages.fetch(page - 1).text.each_line("\n").each_with_object(titles_by_page.fetch(page).dup) do |line, matches|
+            normalized = normalize(line)
+            recent_lines << [normalized, offset]
+            recent_lines.shift if recent_lines.length > MAX_HEADING_LINES
+            offset += line.bytesize
+            next if normalized.empty?
+            match_headings(matches, recent_lines, offset)
+          end.select { |_, location| location }
+        end
+
+        def match_headings(matches, recent_lines, end_byte)
+          recent_lines.each_index do |index|
+            line, start_byte = recent_lines.fetch(index)
+            next if line.empty?
+            title = recent_lines.drop(index).map(&:first).reject(&:empty?).join(" ")
+            next unless matches.key?(title)
+            matches[title] = matches[title].nil? ? [start_byte, end_byte] : false
+          end
+        end
+
+        def normalize(text)
+          text.unicode_normalize(:nfc).gsub(/[[:space:]]+/, " ").strip
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_limits.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      OutlineLimits = Data.define(:depth, :title_bytes, :text_bytes) do
+        def initialize(depth: 16, title_bytes: 4 * 1024, text_bytes: 32 * 1024 * 1024)
+          super
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_span.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      OutlineSpan = Data.define(:title, :range, :parent_index)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/page_boundary.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class PageBoundary
+        def initialize(pages)
+          @pages = pages
+        end
+
+        def match?(entry)
+          return false if entry.title.strip.match?(/\A[[:space:]]*\z/)
+          destination = entry.destination
+          destination.kind == :local && page_top?(destination)
+        end
+
+        private
+
+        def page_top?(destination)
+          return true if %i[Fit FitB FitV FitBV].include?(destination.fit)
+          return true if destination.fit == :XYZ && destination.parameters == [nil, nil, nil]
+
+          top = vertical_coordinate(destination)
+          crop = @pages.fetch(destination.page_number - 1).crop_box
+          top && crop && top >= crop.top
+        end
+
+        def vertical_coordinate(destination)
+          case destination.fit
+          when :XYZ then destination.parameters[1]
+          when :FitH, :FitBH then destination.parameters[0]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/resolve_outline.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class ResolveOutline < Command
+        def initialize(content:, limits: OutlineLimits.new, page_separator: content.page_separator)
+          @content = content
+          @limits = limits
+          @page_separator = page_separator
+        end
+
+        def call
+          resolve_spans
+        end
+
+        private
+
+        Anchor = Data.define(:entry, :start, :heading_end)
+        private_constant :Anchor
+
+        attr_reader :content, :limits, :page_separator, :remaining_subsection_bytes
+
+        def resolve_spans
+          @page_separator = @page_separator.dup.freeze unless @page_separator.frozen?
+          roots = root_entries.map { |entry| root_anchor(entry) }
+          success(payload: reliable_roots?(roots) ? outline_spans(roots) : nil)
+        end
+
+        def root_entries
+          return [] unless content.outline.status == :complete
+          entries_by_parent.fetch(nil, [])
+        end
+
+        def reliable_roots?(roots)
+          return false if roots.empty? || roots.any?(&:nil?)
+          roots.first.start == [1, 0] && ordered?(roots.map(&:start))
+        end
+
+        def outline_spans(roots)
+          @remaining_subsection_bytes = limits.text_bytes
+          pending = sibling_work(roots, [content.pages.length + 1, 0], nil, 0)
+          [].tap do |spans|
+            while (work = pending.pop)
+              anchor, finish, parent_index, depth = work
+              span_index = spans.length
+              spans << build_span(anchor, finish, parent_index)
+              next if depth >= limits.depth
+
+              children = admitted_children(anchor, finish)
+              pending.concat(sibling_work(children, finish, span_index, depth + 1)) if children
+            end
+          end.freeze
+        end
+
+        def sibling_work(anchors, parent_end, parent_index, depth)
+          anchors.each_index.reverse_each.map do |index|
+            [anchors.fetch(index), anchors[index + 1]&.start || parent_end, parent_index, depth]
+          end
+        end
+
+        def build_span(anchor, finish, parent_index)
+          range = TextRange.new(start: anchor.start, finish: finish, page_separator: page_separator)
+          OutlineSpan.new(title: anchor.entry.title.strip.freeze, range: range, parent_index: parent_index)
+        end
+
+        def admitted_children(anchor, finish)
+          entries = entries_by_parent.fetch(anchor.entry.id, [])
+          return if entries.empty?
+          children = entries.map { |entry| child_anchor(entry) }
+          return unless reliable_children?(children, anchor.heading_end || anchor.start, finish)
+          bytes = represented_bytes(children, finish)
+          return if bytes > remaining_subsection_bytes
+
+          @remaining_subsection_bytes -= bytes
+          children
+        end
+
+        def reliable_children?(children, first_allowed_start, parent_end)
+          return false if children.any?(&:nil?)
+          starts = children.map(&:start)
+          (starts.first <=> first_allowed_start) >= 0 && ordered?([*starts, parent_end]) &&
+            children.zip([*starts.drop(1), parent_end]).all? { |child, finish| (child.heading_end <=> finish) <= 0 }
+        end
+
+        def ordered?(positions)
+          positions.each_cons(2).all? { |left, right| (left <=> right).negative? }
+        end
+
+        def represented_bytes(children, finish)
+          boundaries = [*children.drop(1).map(&:start), finish]
+          omitted_separators = boundaries.count { |position| position.last.zero? }
+          absolute_offset(finish) - absolute_offset(children.first.start) - omitted_separators * page_separator.bytesize
+        end
+
+        def absolute_offset(position)
+          page, offset = position
+          page_offsets.fetch(page - 1) + offset
+        end
+
+        def page_offsets
+          @page_offsets ||= content.pages.each_with_object([0]) do |page, offsets|
+            offsets << offsets.last + page.text.bytesize + page_separator.bytesize
+          end
+        end
+
+        def entries_by_parent
+          @entries_by_parent ||= content.outline.entries.group_by(&:parent_id)
+        end
+
+        def page_boundary
+          @page_boundary ||= PageBoundary.new(content.pages)
+        end
+
+        def heading_index
+          @heading_index ||= HeadingIndex.new(pages: content.pages, entries: subsection_entries)
+        end
+
+        def subsection_entries
+          content.outline.entries.select { |entry| entry.parent_id && subsection_destination?(entry) }
+        end
+
+        def root_anchor(entry)
+          return unless page_boundary.match?(entry)
+          Anchor.new(entry: entry, start: [entry.destination.page_number, 0], heading_end: nil)
+        end
+
+        def child_anchor(entry)
+          return unless subsection_destination?(entry)
+          page = entry.destination.page_number
+          heading = heading_index.find(page, entry.title)
+          return unless heading
+          start_byte, end_byte = heading
+          Anchor.new(entry: entry, start: [page, start_byte], heading_end: [page, end_byte])
+        end
+
+        def subsection_destination?(entry)
+          entry.title.bytesize <= limits.title_bytes && page_boundary.match?(entry)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/text_range.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class TextRange < Data.define(:start, :finish, :page_separator)
+        Location = Data.define(:page, :start_byte, :end_byte)
+        Slice = Data.define(:text, :locations)
+
+        def initialize(start:, finish:, page_separator: Content::DEFAULT_PAGE_SEPARATOR)
+          page_separator = page_separator.dup.freeze unless page_separator.frozen?
+          super(start: start.dup.freeze, finish: finish.dup.freeze, page_separator: page_separator)
+        end
+
+        def slice(page_texts:, page_separator: self.page_separator)
+          (start.first..last_page).each_with_object(Slice.new(text: +"", locations: [])) do |number, slice|
+            append_page(slice, number, page_texts.fetch(number - 1), page_separator)
+          end.tap do |slice|
+            slice.text.freeze
+            slice.locations.freeze
+          end
+        end
+
+        private
+
+        def append_page(slice, number, page, page_separator)
+          from = (number == start.first) ? start.last : 0
+          to = (number == finish.first) ? finish.last : page.bytesize
+          slice.text << page_separator unless number == start.first
+          slice.text << page.byteslice(from...to)
+          slice.locations << location(number, from, to, page.bytesize)
+        end
+
+        def last_page
+          finish.last.zero? ? finish.first - 1 : finish.first
+        end
+
+        def location(page, from, to, size)
+          return Location.new(page: page, start_byte: nil, end_byte: nil) if from.zero? && to == size
+          Location.new(page: page, start_byte: from, end_byte: to)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Substrate
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: substrate
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Dan Frenette
@@ -106,6 +106,7 @@ files:
 - lib/substrate/mediums/pdf/document.rb
 - lib/substrate/mediums/pdf/encrypted_pdf.rb
 - lib/substrate/mediums/pdf/entry.rb
+- lib/substrate/mediums/pdf/heading_index.rb
 - lib/substrate/mediums/pdf/invalid_pdf.rb
 - lib/substrate/mediums/pdf/invalid_text.rb
 - lib/substrate/mediums/pdf/limit_exceeded.rb
@@ -114,11 +115,16 @@ files:
 - lib/substrate/mediums/pdf/no_selectable_text.rb
 - lib/substrate/mediums/pdf/outline.rb
 - lib/substrate/mediums/pdf/outline_facts.rb
+- lib/substrate/mediums/pdf/outline_limits.rb
 - lib/substrate/mediums/pdf/outline_parser.rb
+- lib/substrate/mediums/pdf/outline_span.rb
 - lib/substrate/mediums/pdf/page.rb
+- lib/substrate/mediums/pdf/page_boundary.rb
 - lib/substrate/mediums/pdf/page_extractor.rb
 - lib/substrate/mediums/pdf/reader.rb
+- lib/substrate/mediums/pdf/resolve_outline.rb
 - lib/substrate/mediums/pdf/text_extractor.rb
+- lib/substrate/mediums/pdf/text_range.rb
 - lib/substrate/primitives/article_content.rb
 - lib/substrate/primitives/extraction.rb
 - lib/substrate/primitives/metadata.rb