substrate 0.1.0 → 0.2.0

data/lib/substrate/mediums/pdf/document.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class Document
+        def initialize(path, source_ref:)
+          @path = path
+          @source_ref = source_ref
+        end
+
+        def chapters
+          outline_parser.chapters
+        end
+
+        def page_count
+          reader.page_count
+        end
+
+        def page_text(number)
+          page_extractor.text_for_page(number)
+        end
+
+        def pages_text(range)
+          page_extractor.text_for_pages(range)
+        end
+
+        private
+
+        attr_reader :path, :source_ref
+
+        def reader
+          @reader ||= Reader.new(path)
+        end
+
+        def outline_parser
+          @outline_parser ||= OutlineParser.new(reader, source_ref)
+        end
+
+        def page_extractor
+          @page_extractor ||= PageExtractor.new(reader)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/encrypted_pdf.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class EncryptedPdf < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/entry.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Entry = Data.define(:id, :parent_id, :title, :destination)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/invalid_pdf.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class InvalidPdf < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/invalid_text.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class InvalidText < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/limit_exceeded.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class LimitExceeded < StandardError
+        attr_reader :dimension, :limit
+
+        def initialize(dimension, limit)
+          @dimension = dimension
+          @limit = limit
+          super("PDF exceeds #{dimension} limit of #{limit}")
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/limits.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Limits = Data.define(:input_bytes, :pages, :text_bytes, :outline_entries) do
+        def initialize(input_bytes: 25 * 1024 * 1024, pages: 2000,
+          text_bytes: 32 * 1024 * 1024, outline_entries: 2000)
+          values = {input_bytes: input_bytes, pages: pages, text_bytes: text_bytes, outline_entries: outline_entries}
+          raise ArgumentError, "PDF limits must be positive integers" unless values.values.all? { |value| value.is_a?(Integer) && value.positive? }
+          super(**values)
+        end
+
+        def check!(dimension, actual)
+          maximum = public_send(dimension)
+          raise LimitExceeded.new(dimension, maximum) if actual > maximum
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/named_destinations.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class NamedDestinations
+        Lookup = Data.define(:value, :reason)
+
+        def initialize(document, limits)
+          @document = document
+          @limit = limits.outline_entries
+        end
+
+        def lookup(name)
+          Lookup.new(value: named_destination(name), reason: nil)
+        rescue LookupFailure => error
+          Lookup.new(value: nil, reason: error.reason)
+        end
+
+        private
+
+        def name_matches?(key, name)
+          return false unless key.is_a?(String) || key.is_a?(Symbol)
+          text = key.to_s.encode(Encoding::UTF_8)
+          text.valid_encoding? && !text.include?("\0") && text == name
+        rescue EncodingError
+          false
+        end
+
+        class LookupFailure < StandardError
+          attr_reader :reason
+
+          def initialize(reason)
+            @reason = reason
+            super(reason.to_s)
+          end
+        end
+
+        def named_destination(name)
+          if name.is_a?(Symbol)
+            old = @document.wrap(@document.catalog[:Dests])
+            old[name] if old.value.is_a?(Hash)
+          else
+            names = @document.wrap(@document.catalog[:Names])
+            find_name(names[:Dests], name) if names.value.is_a?(Hash) && names[:Dests]
+          end
+        end
+
+        def find_name(root, name)
+          pending = [root]
+          seen = {}
+          work = 0
+          until pending.empty?
+            node = name_tree_node(pending.pop, seen)
+            work = count_work(work)
+            pairs = name_pairs(node[:Names])
+            pairs.each_slice(2) do |key, value|
+              work = count_work(work)
+              return value if name_matches?(key, name)
+            end
+            pending.concat(name_children(node[:Kids]).reverse)
+          end
+          nil
+        end
+
+        def name_tree_node(raw, seen)
+          node = @document.wrap(@document.deref(raw))
+          raise LookupFailure, :malformed_name_tree unless node.value.is_a?(Hash)
+          raise LookupFailure, :destination_lookup_cycle if seen[node.value.object_id]
+          seen[node.value.object_id] = true
+          node
+        end
+
+        def name_pairs(raw)
+          pairs = name_children(raw)
+          raise LookupFailure, :malformed_name_tree if pairs.size.odd?
+          pairs
+        end
+
+        def name_children(raw)
+          return [] unless raw
+          array = @document.wrap(@document.deref(raw))
+          raise LookupFailure, :malformed_name_tree unless array.value.is_a?(Array)
+          array.to_a
+        end
+
+        def count_work(work)
+          raise LookupFailure, :destination_lookup_limit if work >= @limit
+          work + 1
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/no_selectable_text.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class NoSelectableText < StandardError; end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Outline = Data.define(:entries, :status, :reason)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_facts.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class OutlineFacts
+        LOOKUP_FAILURES = [:destination_lookup_cycle, :destination_lookup_limit, :malformed_name_tree].freeze
+
+        def initialize(document, pages, limits)
+          @document = document
+          @destinations = DestinationResolver.new(document, pages, limits)
+          @limit = limits.outline_entries
+          @entries = []
+          @reason = nil
+        end
+
+        def call
+          read_outline
+          publish_outline
+        end
+
+        private
+
+        def read_outline
+          root = @document.catalog[:Outlines]
+          return if root.nil?
+          root = outline_node(root)
+          validate_children(root)
+          traverse(root)
+        rescue HexaPDF::Error
+          degrade(:malformed_outline)
+        end
+
+        def traverse(root)
+          pending = [[root[:First], nil, root[:Last]]]
+          seen = {}
+          until pending.empty?
+            raw, parent_id, last = pending.pop
+            next if raw.nil?
+            item = outline_node(raw)
+            return degrade(:cycle_detected) if seen[item.value.object_id]
+            return degrade(:outline_entries_limit) if @entries.size >= @limit
+            seen[item.value.object_id] = true
+            entry = append_entry(item, parent_id)
+            schedule_links(pending, item, entry, last)
+          end
+        end
+
+        def outline_node(raw)
+          node = @document.deref(raw)
+          node = @document.wrap(node) unless node.is_a?(HexaPDF::Object)
+          raise HexaPDF::Error unless node.value.is_a?(Hash)
+          node
+        end
+
+        def append_entry(item, parent_id)
+          Entry.new(id: @entries.size, parent_id: parent_id,
+            title: outline_title(item), destination: destination(item)).tap { |entry| @entries << entry }
+        end
+
+        def outline_title(item)
+          title = item[:Title]
+          title = title.to_s if title.is_a?(Symbol)
+          raise HexaPDF::Error unless title.is_a?(String)
+          title = title.encode(Encoding::UTF_8)
+          raise HexaPDF::Error unless title.valid_encoding? && !title.include?("\0")
+          title.dup.freeze
+        rescue EncodingError
+          raise HexaPDF::Error
+        end
+
+        def destination(item)
+          fact = @destinations.resolve(item, prior_reason: @reason)
+          degrade(fact.reason) if LOOKUP_FAILURES.include?(fact.reason)
+          fact
+        end
+
+        def schedule_links(pending, item, entry, last)
+          validate_children(item)
+          following = item[:Next]
+          outline_node(following) unless following.nil?
+          validate_last(item, following, last) unless last.nil?
+          pending << [following, entry.parent_id, last]
+          pending << [item[:First], entry.id, item[:Last]]
+        end
+
+        def validate_children(node)
+          first, last, count = node[:First], node[:Last], node[:Count]
+          [first, last].each { |link| outline_node(link) unless link.nil? }
+          raise HexaPDF::Error unless count.nil? || count.is_a?(Integer)
+          raise HexaPDF::Error if first.nil? && (!last.nil? || (count && count != 0))
+        end
+
+        def validate_last(item, following, last)
+          is_last = item.value.equal?(outline_node(last).value)
+          raise HexaPDF::Error if following.nil? != is_last
+        end
+
+        def publish_outline
+          status = if @reason
+            @entries.empty? ? :unavailable : :incomplete
+          else
+            :complete
+          end
+          Outline.new(entries: @entries.freeze, status: status, reason: @reason)
+        end
+
+        def degrade(reason)
+          @reason ||= reason
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/outline_parser.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class OutlineParser
+        def initialize(reader, source_ref)
+          @reader = reader
+          @source_ref = source_ref
+        end
+
+        def chapters
+          items = extract_items
+          items.each_with_index.map { |item, index| to_chapter(item, index, items) }
+        end
+
+        private
+
+        attr_reader :reader, :source_ref
+
+        def extract_items
+          reader.outline_items.each_with_object([]) do |(item, _level), entries|
+            entry = parse_item(item)
+            entries << entry if entry
+          end
+        end
+
+        def parse_item(item)
+          title = item[:Title]
+          page = resolve_page(item)
+          return nil if title.nil? || title.empty? || page.nil?
+
+          {id: chapter_id(title, page), title: title, page: page}
+        end
+
+        def resolve_page(item)
+          dest = item.destination || goto_destination(item)
+          return nil unless dest
+
+          page_obj = dest.first
+          reader.page_index(page_obj)&.succ
+        end
+
+        def goto_destination(item)
+          action = item[:A]
+          return nil unless action
+
+          action[:D] if action[:S] == :GoTo
+        end
+
+        def chapter_id(title, page)
+          Digest::SHA256.hexdigest("#{page}:#{title}")[0, 8]
+        end
+
+        def to_chapter(item, index, all_items)
+          start_page = item[:page]
+          next_start_page = all_items.dig(index + 1, :page)
+          last_page = next_start_page ? [next_start_page.pred, start_page].max : reader.page_count
+
+          Chapter.new(source_ref: source_ref, id: item[:id], title: item[:title], pages: start_page..last_page)
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/page.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Page = Data.define(:number, :text)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/page_extractor.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class PageExtractor
+        def initialize(reader)
+          @reader = reader
+        end
+
+        def text_for_page(number)
+          page = reader.pages[number - 1]
+          return "" unless page
+
+          TextExtractor.extract(page)
+        end
+
+        def text_for_pages(range)
+          range.map { |n| text_for_page(n) }.reject(&:empty?).join("\n\n")
+        end
+
+        private
+
+        attr_reader :reader
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/reader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class Reader
+        def initialize(path, document_class: HexaPDF::Document)
+          @path = path
+          @document_class = document_class
+        end
+
+        def pages
+          document.pages
+        end
+
+        def page_count
+          pages.count
+        end
+
+        def outline_items
+          document.outline.each_item
+        end
+
+        def page_index(page_object)
+          pages.find_index { |page| page == page_object }
+        end
+
+        private
+
+        attr_reader :path, :document_class
+
+        def document
+          @document ||= document_class.open(path, config: {"filter.map" => {BrotliDecode: nil}})
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/text_extractor.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+
+module Substrate
+  module Mediums
+    module Pdf
+      class TextExtractor < HexaPDF::Content::Processor
+        def self.extract(page)
+          new(page.resources).tap { |extractor| page.process_contents(extractor) }.text
+        end
+
+        attr_reader :text
+
+        def initialize(resources)
+          super
+          @text = +""
+        end
+
+        def show_text(str)
+          append(decode_text(str))
+        end
+
+        def show_text_with_positioning(arr)
+          append(decode_text(arr))
+        end
+
+        private
+
+        def append(str)
+          text << str << " "
+        end
+      end
+    end
+  end
+end

data/lib/substrate/primitives/article_content.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Substrate
+  ArticleContent = Data.define(
+    :source_ref,
+    :text,
+    :title,
+    :metadata
+  )
+end

data/lib/substrate/primitives/extraction.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  Extraction = Data.define(:text, :source, :metadata)
+end

data/lib/substrate/primitives/metadata.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Substrate
+  Metadata = Data.define(
+    :requested_url,
+    :final_url,
+    :redirected,
+    :content_type,
+    :extractor
+  ) do
+    def initialize(
+      requested_url:,
+      final_url: nil,
+      redirected: false,
+      content_type: nil,
+      extractor: nil
+    )
+      super
+    end
+  end
+end

data/lib/substrate/primitives/page_collection.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  PageCollection = Data.define(:source_ref, :pages)
+end

data/lib/substrate/primitives/pdf_file.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  PdfFile = Data.define(:path)
+end

data/lib/substrate/primitives/raw_text.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  RawText = Data.define(:text)
+end

data/lib/substrate/primitives/source.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  Source = Data.define(:kind, :locator, :metadata)
+end

data/lib/substrate/primitives/text_content.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  TextContent = Data.define(:source_ref, :text)
+end

data/lib/substrate/primitives/text_file.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Substrate
+  TextFile = Data.define(:path)
+end

data/lib/substrate/primitives/transcripts/content.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Transcripts
+    Content = Data.define(
+      :source_ref,
+      :segments,
+      :metadata
+    )
+  end
+end

data/lib/substrate/primitives/transcripts/metadata.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Transcripts
+    Metadata = Data.define(
+      :provider,
+      :model,
+      :language,
+      :duration_seconds,
+      :diarized,
+      :word_timestamps
+    ) do
+      def initialize(
+        provider:,
+        model:,
+        language: nil,
+        duration_seconds: nil,
+        diarized: false,
+        word_timestamps: false
+      )
+        super
+      end
+    end
+  end
+end

data/lib/substrate/primitives/transcripts/segment.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Transcripts
+    Segment = Data.define(
+      :text,
+      :start,
+      :end,
+      :confidence,
+      :speaker,
+      :words
+    ) do
+      def initialize(
+        text:,
+        start:,
+        end:,
+        confidence: nil,
+        speaker: nil,
+        words: []
+      )
+        super
+      end
+    end
+  end
+end