substrate 0.1.0 → 0.3.0

data/lib/substrate/mediums/pdf/destination_resolver.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class DestinationResolver
+        FIT_ARITY = {XYZ: 3, Fit: 0, FitH: 1, FitV: 1, FitR: 4, FitB: 0, FitBH: 1, FitBV: 1}.freeze
+
+        def initialize(document, pages, limits)
+          @document = document
+          @pages = pages
+          @names = NamedDestinations.new(document, limits)
+          @limit = limits.outline_entries
+        end
+
+        def resolve(item, prior_reason:)
+          return local_destination(item[:Dest], prior_reason) if item[:Dest]
+          return fact(kind: :none) unless item[:A]
+          action_destination(item[:A], prior_reason)
+        end
+
+        private
+
+        def action_destination(raw, prior_reason)
+          action = @document.wrap(@document.deref(raw))
+          return fact(reason: :malformed_action) unless action.value.is_a?(Hash)
+          case action[:S]
+          when :GoTo then local_destination(action[:D], prior_reason)
+          when :GoToR then remote_destination(action)
+          else fact(reason: :unsupported_action)
+          end
+        end
+
+        def local_destination(raw, prior_reason)
+          values, name, reason = resolve_aliases(raw, prior_reason)
+          return fact(name: name, reason: reason) if reason
+          view = destination_view(values).with(name: name)
+          return view if view.reason
+          page = @document.deref(values.first)
+          index = @pages.index { |candidate| candidate == page }
+          index ? view.with(kind: :local, page_number: index + 1) : view.with(reason: :page_not_found)
+        end
+
+        def resolve_aliases(raw, prior_reason)
+          seen = {}
+          name = nil
+          loop do
+            value = destination_value(raw)
+            return [value, name, nil] unless named?(value)
+            text = destination_name(value)
+            return [nil, nil, :invalid_destination_name] unless text
+            name ||= text
+            reason = alias_failure(seen, value, text)
+            return [nil, name, reason] if reason
+            lookup = @names.lookup(value)
+            return [nil, name, prior_reason || lookup.reason || :destination_not_found] unless lookup.value
+            raw = lookup.value
+          end
+        end
+
+        def alias_failure(seen, value, text)
+          key = [value.class, text]
+          return :destination_lookup_cycle if seen[key]
+          return :destination_lookup_limit if seen.size >= @limit
+          seen[key] = true
+          nil
+        end
+
+        def destination_view(values)
+          return fact(reason: :malformed_destination) unless values.is_a?(Array)
+          fit, parameters = values[1], values.drop(2)
+          valid_parameters = parameters.all? { |value| value.nil? || (value.is_a?(Numeric) && value.finite?) }
+          valid = FIT_ARITY[fit] == parameters.size && valid_parameters
+          fact(fit: fit.is_a?(Symbol) ? fit : nil, parameters: valid_parameters ? parameters.freeze : [].freeze,
+            reason: valid ? nil : :invalid_destination_view)
+        end
+
+        def remote_destination(action)
+          file = remote_file(action[:F])
+          target = {action: :GoToR, file: file, page_index: nil, name: nil}.freeze
+          destination = remote_location(destination_value(action[:D]), target)
+          file ? destination : destination.with(kind: :unresolved, reason: :invalid_remote_target)
+        end
+
+        def destination_value(raw)
+          object = @document.wrap(@document.deref(raw))
+          object = @document.wrap(@document.deref(object[:D])) if object.value.is_a?(Hash)
+          object.value.is_a?(Array) ? object.to_a : object.value
+        end
+
+        def destination_name(value)
+          return unless named?(value)
+          text = value.to_s.encode(Encoding::UTF_8)
+          text.dup.freeze if text.valid_encoding? && !text.include?("\0")
+        rescue EncodingError
+          nil
+        end
+
+        def named?(value)
+          value.is_a?(String) || value.is_a?(Symbol)
+        end
+
+        def remote_file(raw)
+          file = @document.wrap(@document.deref(raw))
+          value = file.value.is_a?(Hash) ? file[:UF] || file[:F] : file.value
+          destination_name(value)
+        end
+
+        def remote_location(value, target)
+          return remote_name(value, target) if named?(value)
+          view = destination_view(value).with(remote_target: target)
+          return view if view.reason
+          index = value.first
+          return view.with(reason: :invalid_remote_page) unless index.is_a?(Integer) && index >= 0
+          view.with(kind: :remote, remote_target: target.merge(page_index: index).freeze)
+        end
+
+        def remote_name(value, target)
+          name = destination_name(value)
+          return fact(remote_target: target, reason: :invalid_destination_name) unless name
+          fact(kind: :remote, name: name, remote_target: target.merge(name: name).freeze)
+        end
+
+        def fact(kind: :unresolved, page_number: nil, name: nil, fit: nil, parameters: [].freeze, remote_target: nil, reason: nil)
+          Destination.new(kind: kind, page_number: page_number, name: name, fit: fit,
+            parameters: parameters, remote_target: remote_target, reason: reason)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/document.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class Document
+        def initialize(path, source_ref:)
+          @path = path
+          @source_ref = source_ref
+        end
+
+        def chapters
+          outline_parser.chapters
+        end
+
+        def page_count
+          reader.page_count
+        end
+
+        def page_text(number)
+          page_extractor.text_for_page(number)
+        end
+
+        def pages_text(range)
+          page_extractor.text_for_pages(range)
+        end
+
+        private
+
+        attr_reader :path, :source_ref
+
+        def reader
+          @reader ||= Reader.new(path)
+        end
+
+        def outline_parser
+          @outline_parser ||= OutlineParser.new(reader, source_ref)
+        end
+
+        def page_extractor
+          @page_extractor ||= PageExtractor.new(reader)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/encrypted_pdf.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class EncryptedPdf < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Entry = Data.define(:id, :parent_id, :title, :destination)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/invalid_pdf.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class InvalidPdf < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/invalid_text.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class InvalidText < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/limit_exceeded.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class LimitExceeded < StandardError
+        attr_reader :dimension, :limit
+
+        def initialize(dimension, limit)
+          @dimension = dimension
+          @limit = limit
+          super("PDF exceeds #{dimension} limit of #{limit}")
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/limits.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Limits = Data.define(:input_bytes, :pages, :text_bytes, :outline_entries) do
+        def initialize(input_bytes: 25 * 1024 * 1024, pages: 2000,
+          text_bytes: 32 * 1024 * 1024, outline_entries: 2000)
+          values = {input_bytes: input_bytes, pages: pages, text_bytes: text_bytes, outline_entries: outline_entries}
+          raise ArgumentError, "PDF limits must be positive integers" unless values.values.all? { |value| value.is_a?(Integer) && value.positive? }
+          super(**values)
+        end
+
+        def check!(dimension, actual)
+          maximum = public_send(dimension)
+          raise LimitExceeded.new(dimension, maximum) if actual > maximum
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/named_destinations.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class NamedDestinations
+        Lookup = Data.define(:value, :reason)
+
+        def initialize(document, limits)
+          @document = document
+          @limit = limits.outline_entries
+        end
+
+        def lookup(name)
+          Lookup.new(value: named_destination(name), reason: nil)
+        rescue LookupFailure => error
+          Lookup.new(value: nil, reason: error.reason)
+        end
+
+        private
+
+        def name_matches?(key, name)
+          return false unless key.is_a?(String) || key.is_a?(Symbol)
+          text = key.to_s.encode(Encoding::UTF_8)
+          text.valid_encoding? && !text.include?("\0") && text == name
+        rescue EncodingError
+          false
+        end
+
+        class LookupFailure < StandardError
+          attr_reader :reason
+
+          def initialize(reason)
+            @reason = reason
+            super(reason.to_s)
+          end
+        end
+
+        def named_destination(name)
+          if name.is_a?(Symbol)
+            old = @document.wrap(@document.catalog[:Dests])
+            old[name] if old.value.is_a?(Hash)
+          else
+            names = @document.wrap(@document.catalog[:Names])
+            find_name(names[:Dests], name) if names.value.is_a?(Hash) && names[:Dests]
+          end
+        end
+
+        def find_name(root, name)
+          pending = [root]
+          seen = {}
+          work = 0
+          until pending.empty?
+            node = name_tree_node(pending.pop, seen)
+            work = count_work(work)
+            pairs = name_pairs(node[:Names])
+            pairs.each_slice(2) do |key, value|
+              work = count_work(work)
+              return value if name_matches?(key, name)
+            end
+            pending.concat(name_children(node[:Kids]).reverse)
+          end
+          nil
+        end
+
+        def name_tree_node(raw, seen)
+          node = @document.wrap(@document.deref(raw))
+          raise LookupFailure, :malformed_name_tree unless node.value.is_a?(Hash)
+          raise LookupFailure, :destination_lookup_cycle if seen[node.value.object_id]
+          seen[node.value.object_id] = true
+          node
+        end
+
+        def name_pairs(raw)
+          pairs = name_children(raw)
+          raise LookupFailure, :malformed_name_tree if pairs.size.odd?
+          pairs
+        end
+
+        def name_children(raw)
+          return [] unless raw
+          array = @document.wrap(@document.deref(raw))
+          raise LookupFailure, :malformed_name_tree unless array.value.is_a?(Array)
+          array.to_a
+        end
+
+        def count_work(work)
+          raise LookupFailure, :destination_lookup_limit if work >= @limit
+          work + 1
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/no_selectable_text.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class NoSelectableText < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Outline = Data.define(:entries, :status, :reason)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_facts.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class OutlineFacts
+        LOOKUP_FAILURES = [:destination_lookup_cycle, :destination_lookup_limit, :malformed_name_tree].freeze
+
+        def initialize(document, pages, limits)
+          @document = document
+          @destinations = DestinationResolver.new(document, pages, limits)
+          @limit = limits.outline_entries
+          @entries = []
+          @reason = nil
+        end
+
+        def call
+          read_outline
+          publish_outline
+        end
+
+        private
+
+        def read_outline
+          root = @document.catalog[:Outlines]
+          return if root.nil?
+          root = outline_node(root)
+          validate_children(root)
+          traverse(root)
+        rescue HexaPDF::Error
+          degrade(:malformed_outline)
+        end
+
+        def traverse(root)
+          pending = [[root[:First], nil, root[:Last]]]
+          seen = {}
+          until pending.empty?
+            raw, parent_id, last = pending.pop
+            next if raw.nil?
+            item = outline_node(raw)
+            return degrade(:cycle_detected) if seen[item.value.object_id]
+            return degrade(:outline_entries_limit) if @entries.size >= @limit
+            seen[item.value.object_id] = true
+            entry = append_entry(item, parent_id)
+            schedule_links(pending, item, entry, last)
+          end
+        end
+
+        def outline_node(raw)
+          node = @document.deref(raw)
+          node = @document.wrap(node) unless node.is_a?(HexaPDF::Object)
+          raise HexaPDF::Error unless node.value.is_a?(Hash)
+          node
+        end
+
+        def append_entry(item, parent_id)
+          Entry.new(id: @entries.size, parent_id: parent_id,
+            title: outline_title(item), destination: destination(item)).tap { |entry| @entries << entry }
+        end
+
+        def outline_title(item)
+          title = item[:Title]
+          title = title.to_s if title.is_a?(Symbol)
+          raise HexaPDF::Error unless title.is_a?(String)
+          title = title.encode(Encoding::UTF_8)
+          raise HexaPDF::Error unless title.valid_encoding? && !title.include?("\0")
+          title.dup.freeze
+        rescue EncodingError
+          raise HexaPDF::Error
+        end
+
+        def destination(item)
+          fact = @destinations.resolve(item, prior_reason: @reason)
+          degrade(fact.reason) if LOOKUP_FAILURES.include?(fact.reason)
+          fact
+        end
+
+        def schedule_links(pending, item, entry, last)
+          validate_children(item)
+          following = item[:Next]
+          outline_node(following) unless following.nil?
+          validate_last(item, following, last) unless last.nil?
+          pending << [following, entry.parent_id, last]
+          pending << [item[:First], entry.id, item[:Last]]
+        end
+
+        def validate_children(node)
+          first, last, count = node[:First], node[:Last], node[:Count]
+          [first, last].each { |link| outline_node(link) unless link.nil? }
+          raise HexaPDF::Error unless count.nil? || count.is_a?(Integer)
+          raise HexaPDF::Error if first.nil? && (!last.nil? || (count && count != 0))
+        end
+
+        def validate_last(item, following, last)
+          is_last = item.value.equal?(outline_node(last).value)
+          raise HexaPDF::Error if following.nil? != is_last
+        end
+
+        def publish_outline
+          status = if @reason
+            @entries.empty? ? :unavailable : :incomplete
+          else
+            :complete
+          end
+          Outline.new(entries: @entries.freeze, status: status, reason: @reason)
+        end
+
+        def degrade(reason)
+          @reason ||= reason
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_parser.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class OutlineParser
+        def initialize(reader, source_ref)
+          @reader = reader
+          @source_ref = source_ref
+        end
+
+        def chapters
+          items = extract_items
+          items.each_with_index.map { |item, index| to_chapter(item, index, items) }
+        end
+
+        private
+
+        attr_reader :reader, :source_ref
+
+        def extract_items
+          reader.outline_items.each_with_object([]) do |(item, _level), entries|
+            entry = parse_item(item)
+            entries << entry if entry
+          end
+        end
+
+        def parse_item(item)
+          title = item[:Title]
+          page = resolve_page(item)
+          return nil if title.nil? || title.empty? || page.nil?
+
+          {id: chapter_id(title, page), title: title, page: page}
+        end
+
+        def resolve_page(item)
+          dest = item.destination || goto_destination(item)
+          return nil unless dest
+
+          page_obj = dest.first
+          reader.page_index(page_obj)&.succ
+        end
+
+        def goto_destination(item)
+          action = item[:A]
+          return nil unless action
+
+          action[:D] if action[:S] == :GoTo
+        end
+
+        def chapter_id(title, page)
+          Digest::SHA256.hexdigest("#{page}:#{title}")[0, 8]
+        end
+
+        def to_chapter(item, index, all_items)
+          start_page = item[:page]
+          next_start_page = all_items.dig(index + 1, :page)
+          last_page = next_start_page ? [next_start_page.pred, start_page].max : reader.page_count
+
+          Chapter.new(source_ref: source_ref, id: item[:id], title: item[:title], pages: start_page..last_page)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/page.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Page = Data.define(:number, :text, :crop_box)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/page_extractor.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class PageExtractor
+        def initialize(reader)
+          @reader = reader
+        end
+
+        def text_for_page(number)
+          page = reader.pages[number - 1]
+          return "" unless page
+
+          TextExtractor.extract(page)
+        end
+
+        def text_for_pages(range)
+          range.map { |n| text_for_page(n) }.reject(&:empty?).join("\n\n")
+        end
+
+        private
+
+        attr_reader :reader
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/reader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class Reader
+        def initialize(path, document_class: HexaPDF::Document)
+          @path = path
+          @document_class = document_class
+        end
+
+        def pages
+          document.pages
+        end
+
+        def page_count
+          pages.count
+        end
+
+        def outline_items
+          document.outline.each_item
+        end
+
+        def page_index(page_object)
+          pages.find_index { |page| page == page_object }
+        end
+
+        private
+
+        attr_reader :path, :document_class
+
+        def document
+          @document ||= document_class.open(path, config: {"filter.map" => {BrotliDecode: nil}})
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/text_extractor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class TextExtractor < HexaPDF::Content::Processor
+        def self.extract(page)
+          new(page.resources).tap { |extractor| page.process_contents(extractor) }.text
+        end
+
+        attr_reader :text
+
+        def initialize(resources)
+          super
+          @text = +""
+        end
+
+        def show_text(str)
+          append(decode_text(str))
+        end
+
+        def show_text_with_positioning(arr)
+          append(decode_text(arr))
+        end
+
+        private
+
+        def append(str)
+          text << str << " "
+        end
+      end
+    end
+  end
+end

data/lib/substrate/primitives/article_content.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Substrate
+  ArticleContent = Data.define(
+    :source_ref,
+    :text,
+    :title,
+    :metadata
+  )
+end

data/lib/substrate/primitives/extraction.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  Extraction = Data.define(:text, :source, :metadata)
+end