substrate 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7b3e7c25e9af2d9e3b590493a0b749c1413c8fc4542074a891f0879fb308dd2
-  data.tar.gz: e88153c370e8f419a810c7014b9b5e15790741811ed40874ac5f3517dba0b8c3
+  metadata.gz: 20af64cb9a41e61ad9af8a5f678f3485947689193ad3f71d7638bd332452c88b
+  data.tar.gz: 34bc75ca3736801c7f35d968a626e32a597d62c36ef6516b7f727c1cd21e53ff
 SHA512:
-  metadata.gz: 4200859c9d863f6d18692d9c92d97208cab9b07b17aa9d73f791724917752cc4261709f4fc49216ad2c0a71833d6fec9917c6f0e94972c7287f6f03046888b61
-  data.tar.gz: 4fb43377d34974ce6f02caaef6a2437e34b3a2d21948776f3ca1ba6d5955250365c576c3e9f7776750c95394c35f0806bea45aaa6449198ebf908d101a48ee75
+  metadata.gz: 46c461e08228dcf87eed7d6e6c791619b3a83fc6ee72f1e2804db5e9d81a606d0299afdd3a69c5ce4ba52dc3a3fff1720db1dee30ad1cb810fb7f1bcbcc855d6
+  data.tar.gz: 744c7900ef351097160f26a695c440f1197b07ad9aafd38c06d9ce86a5a6714d445d766d048c1cd8ed69cabef792618e4879b19b5383d599441908d3d59c6447

data/lib/substrate/classify.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Substrate
+  class Classify < Command
+    class UnsupportedSource < StandardError; end
+    class ResolutionError < StandardError; end
+
+    EXTENSION_MAP = {
+      ".txt" => :text_file,
+      ".md" => :text_file,
+      ".rst" => :text_file,
+      ".pdf" => :pdf_file,
+      ".mp3" => :audio_file
+    }.freeze
+
+    def initialize(input:)
+      @input = input
+    end
+
+    def call
+      success(payload: classified)
+    end
+
+    private
+
+    attr_reader :input
+
+    def classified
+      return RawText.new(text: input) unless resolved_path || url?
+
+      return Sources::Url.new(input) if url?
+
+      case file_kind
+      when :pdf_file then Sources::Pdf.new(resolved_path)
+      when :text_file then Sources::Text.new(resolved_path)
+      when :audio_file then Sources::Audio.new(resolved_path)
+      end
+    end
+
+    def url?
+      uri = URI.parse(input)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def file_kind
+      ext = File.extname(resolved_path).downcase
+      EXTENSION_MAP.fetch(ext) do
+        raise UnsupportedSource, "Unsupported file type: #{ext}"
+      end
+    end
+
+    def resolved_path
+      @resolved_path ||= begin
+        result = Detection::PathResolver.call(raw: input)
+        raise ResolutionError, result.error.message unless result.success?
+
+        result.payload
+      end
+    end
+  end
+end

data/lib/substrate/cli/availability.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Cli
+    class Availability
+      def self.call(executable)
+        system("which", executable, out: File::NULL, err: File::NULL)
+      end
+    end
+  end
+end

data/lib/substrate/cli/error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Cli
+    class Error < StandardError; end
+  end
+end

data/lib/substrate/cli/runner.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Substrate
+  module Cli
+    class Runner < Command
+      ExecutionError = Class.new(Error)
+
+      def initialize(argv:)
+        @argv = argv
+      end
+
+      def call
+        stdout, stderr, status = Open3.capture3(*argv)
+
+        raise ExecutionError, stderr unless status.success?
+
+        success(payload: stdout)
+      rescue ExecutionError
+        raise
+      rescue => e
+        raise ExecutionError, e.message
+      end
+
+      private
+
+      attr_reader :argv
+    end
+  end
+end

data/lib/substrate/command.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Substrate
+  Result = Data.define(:payload, :error) do
+    def success? = error.nil?
+    def failure? = !error.nil?
+  end
+
+  class Command
+    def self.call(...)
+      new(...).call
+    rescue => e
+      Result.new(payload: nil, error: e)
+    end
+
+    def success(payload:)
+      Result.new(payload: payload, error: nil)
+    end
+
+    def failure(error)
+      Result.new(payload: nil, error: error)
+    end
+  end
+end

data/lib/substrate/configuration.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Configuration
+    class << self
+      def deepgram_api_key
+        @deepgram_api_key ||= ENV.fetch("DEEPGRAM_API_KEY", nil)
+      end
+
+      attr_writer :deepgram_api_key
+
+      def configure
+        yield self
+      end
+
+      def reset!
+        @deepgram_api_key = nil
+      end
+    end
+  end
+end

data/lib/substrate/detection/path_resolver.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Substrate
+  module Detection
+    class PathResolver < Command
+      def initialize(raw:)
+        @raw = raw
+      end
+
+      def call
+        path = candidates.find { |p| p.file? }
+        success(payload: path&.to_s)
+      end
+
+      private
+
+      attr_reader :raw
+
+      def candidates
+        [raw, shell_unescape].compact.uniq.map { |s| Pathname.new(s).expand_path }
+      end
+
+      def shell_unescape
+        parts = Shellwords.shellsplit(raw)
+        parts.first if parts.length == 1
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/substrate/distill.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Substrate
+  class Distill < Command
+    def initialize(content:)
+      @content = content
+    end
+
+    def call
+      case content
+      in RawText(text:)
+        Handlers::RawText.call(text: text)
+      in TextContent(source_ref:, text:)
+        Handlers::TextFile.call(text: text, source_ref: source_ref)
+      in ArticleContent(source_ref:, text:, title:, metadata:)
+        Handlers::Article.call(content: content)
+      in PageCollection(source_ref:, pages:)
+        document = Mediums::Pdf::Document.new(source_ref.locator, source_ref: source_ref)
+        Handlers::Pdf.call(document: document, pages: pages)
+      in Transcripts::Content(source_ref:, segments:, metadata:)
+        Handlers::Transcript.call(content: content)
+      else
+        failure(ArgumentError.new("Unsupported content type: #{content.class}"))
+      end
+    end
+
+    private
+
+    attr_reader :content
+  end
+end

data/lib/substrate/extract_pdf.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+require "stringio"
+
+module Substrate
+  class ExtractPdf < Command
+    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new)
+      @bytes = bytes
+      @source_ref = source_ref
+      @limits = limits
+    end
+
+    def call
+      validate_input
+      extract_document
+    end
+
+    private
+
+    def validate_input
+      raise ArgumentError, "bytes must be a String" unless @bytes.is_a?(String)
+      @limits.check!(:input_bytes, @bytes.bytesize)
+    end
+
+    def extract_document
+      StringIO.open(@bytes) do |io|
+        document = HexaPDF::Document.new(io: io, config: {"document.auto_decrypt" => false, "filter.map" => {BrotliDecode: nil}})
+        raise Mediums::Pdf::EncryptedPdf, "Encrypted PDFs are not supported" if document.encrypted?
+        objects = page_objects(document)
+        pages = extract_pages(objects)
+        outline = Mediums::Pdf::OutlineFacts.new(document, objects, @limits).call
+        publish_content(pages, outline)
+      end
+    rescue HexaPDF::Error => error
+      raise Mediums::Pdf::InvalidText, "PDF Unicode mapping is invalid", cause: error if error.cause.is_a?(EncodingError)
+      raise Mediums::Pdf::InvalidPdf, "PDF parsing or page extraction failed", cause: error
+    end
+
+    def page_objects(document)
+      root = document.catalog[:Pages]
+      raise Mediums::Pdf::InvalidPdf, "PDF page tree is missing" unless root.is_a?(HexaPDF::Dictionary)
+      count = root[:Count]
+      raise Mediums::Pdf::InvalidPdf, "PDF page count is invalid" unless count.is_a?(Integer) && count >= 0
+      @limits.check!(:pages, count)
+      pending = [[root, nil]]
+      seen = {}
+      [].tap do |pages|
+        until pending.empty?
+          candidate, parent = pending.pop
+          node = document.deref(candidate)
+          raise Mediums::Pdf::InvalidPdf, "PDF page tree node is invalid" unless node.is_a?(HexaPDF::Dictionary)
+          raise Mediums::Pdf::InvalidPdf, "PDF page ancestry is inconsistent" unless node[:Parent] == parent
+          identity = node.indirect? ? [node.oid, node.gen] : node.data.object_id
+          raise Mediums::Pdf::InvalidPdf, "PDF page tree contains a cycle or repeated node" if seen[identity]
+          seen[identity] = true
+          if node[:Type] == :Page
+            @limits.check!(:pages, pages.size + 1)
+            pages << document.wrap(node, type: :Page)
+          elsif node[:Type] == :Pages
+            kids = node[:Kids]
+            raise Mediums::Pdf::InvalidPdf, "PDF page tree children are invalid" unless kids.is_a?(HexaPDF::PDFArray)
+            pending.concat(kids.to_a.reverse.map { |child| [child, node] })
+          else
+            raise Mediums::Pdf::InvalidPdf, "PDF page tree type is invalid"
+          end
+        end
+      end
+    end
+
+    def extract_pages(objects)
+      text_bytes = 0
+      objects.map.with_index do |page, index|
+        @limits.check!(:pages, index + 1)
+        text = page.extract_text.encode(Encoding::UTF_8)
+        raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8 or NUL" unless text.valid_encoding? && !text.include?("\0")
+        text_bytes += text.bytesize + ((index == 0) ? 0 : 2)
+        @limits.check!(:text_bytes, text_bytes)
+        Mediums::Pdf::Page.new(number: index + 1, text: text.freeze)
+      end.freeze.tap do |pages|
+        raise Mediums::Pdf::NoSelectableText, "PDF contains no selectable text" if pages.all? { |page| page.text.match?(/\A[[:space:]]*\z/) }
+      end
+    rescue EncodingError => error
+      raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8", cause: error
+    end
+
+    def publish_content(pages, outline)
+      metadata = {extractor: "HexaPDF", extractor_version: HexaPDF::VERSION.dup.freeze,
+                  input_bytes: @bytes.bytesize, page_count: pages.size,
+                  text_bytes: pages.sum { |page| page.text.bytesize } + 2 * (pages.size - 1)}.freeze
+      success(payload: Mediums::Pdf::Content.new(pages: pages, source_ref: @source_ref,
+        metadata: metadata, outline: outline))
+    end
+  end
+end

data/lib/substrate/handlers/article.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Article < Command
+      def initialize(content:)
+        @content = content
+      end
+
+      def call
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :content
+
+      def extraction
+        Extraction.new(
+          text: content.text,
+          source: source,
+          metadata: extraction_metadata
+        )
+      end
+
+      def source
+        Source.new(
+          kind: :article,
+          locator: content.source_ref.locator,
+          metadata: content.metadata
+        )
+      end
+
+      def extraction_metadata
+        {title: content.title}.compact
+      end
+    end
+  end
+end

data/lib/substrate/handlers/pdf.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Pdf < Command
+      def initialize(document:, pages: nil)
+        @document = document
+        @pages = pages
+      end
+
+      def call
+        range = pages || (1..document.page_count)
+        text = document.pages_text(range)
+        source = Source.new(kind: :pdf_file, locator: nil, metadata: {
+          page_count: document.page_count
+        })
+        extraction = Extraction.new(text: text, source: source, metadata: {
+          pages_extracted: range
+        })
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :document, :pages
+    end
+  end
+end

data/lib/substrate/handlers/raw_text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class RawText < Command
+      def initialize(text:)
+        @text = text
+      end
+
+      def call
+        source = Source.new(kind: :raw_text, locator: nil, metadata: {})
+        extraction = Extraction.new(text: text, source: source, metadata: {})
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :text
+    end
+  end
+end

data/lib/substrate/handlers/text_file.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class TextFile < Command
+      def initialize(text:, source_ref:)
+        @text = text
+        @source_ref = source_ref
+      end
+
+      def call
+        source = Source.new(kind: :text_file, locator: source_ref.locator, metadata: {
+          line_count: text.count("\n"),
+          file_size: File.size(source_ref.locator)
+        })
+        extraction = Extraction.new(text: text, source: source, metadata: {})
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :text, :source_ref
+    end
+  end
+end

data/lib/substrate/handlers/transcript.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Transcript < Command
+      def initialize(content:)
+        @content = content
+      end
+
+      def call
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :content
+
+      def extraction
+        Extraction.new(
+          text: full_text,
+          source: source,
+          metadata: extraction_metadata
+        )
+      end
+
+      def full_text
+        content.segments.map(&:text).join("\n\n")
+      end
+
+      def source
+        Source.new(
+          kind: :audio,
+          locator: content.source_ref.locator,
+          metadata: content.metadata
+        )
+      end
+
+      def extraction_metadata
+        {
+          segment_count: content.segments.count,
+          duration_seconds: content.metadata.duration_seconds,
+          language: content.metadata.language,
+          diarized: content.metadata.diarized
+        }.compact
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/chapter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Chapter = Data.define(:source_ref, :id, :title, :pages)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/content.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Content = Data.define(:pages, :source_ref, :metadata, :outline) do
+        def text
+          pages.map(&:text).join("\n\n").freeze
+        end
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/destination.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Destination = Data.define(:kind, :page_number, :name, :fit, :parameters, :remote_target, :reason)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/destination_resolver.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      class DestinationResolver
+        FIT_ARITY = {XYZ: 3, Fit: 0, FitH: 1, FitV: 1, FitR: 4, FitB: 0, FitBH: 1, FitBV: 1}.freeze
+
+        def initialize(document, pages, limits)
+          @document = document
+          @pages = pages
+          @names = NamedDestinations.new(document, limits)
+          @limit = limits.outline_entries
+        end
+
+        def resolve(item, prior_reason:)
+          return local_destination(item[:Dest], prior_reason) if item[:Dest]
+          return fact(kind: :none) unless item[:A]
+          action_destination(item[:A], prior_reason)
+        end
+
+        private
+
+        def action_destination(raw, prior_reason)
+          action = @document.wrap(@document.deref(raw))
+          return fact(reason: :malformed_action) unless action.value.is_a?(Hash)
+          case action[:S]
+          when :GoTo then local_destination(action[:D], prior_reason)
+          when :GoToR then remote_destination(action)
+          else fact(reason: :unsupported_action)
+          end
+        end
+
+        def local_destination(raw, prior_reason)
+          values, name, reason = resolve_aliases(raw, prior_reason)
+          return fact(name: name, reason: reason) if reason
+          view = destination_view(values).with(name: name)
+          return view if view.reason
+          page = @document.deref(values.first)
+          index = @pages.index { |candidate| candidate == page }
+          index ? view.with(kind: :local, page_number: index + 1) : view.with(reason: :page_not_found)
+        end
+
+        def resolve_aliases(raw, prior_reason)
+          seen = {}
+          name = nil
+          loop do
+            value = destination_value(raw)
+            return [value, name, nil] unless named?(value)
+            text = destination_name(value)
+            return [nil, nil, :invalid_destination_name] unless text
+            name ||= text
+            reason = alias_failure(seen, value, text)
+            return [nil, name, reason] if reason
+            lookup = @names.lookup(value)
+            return [nil, name, prior_reason || lookup.reason || :destination_not_found] unless lookup.value
+            raw = lookup.value
+          end
+        end
+
+        def alias_failure(seen, value, text)
+          key = [value.class, text]
+          return :destination_lookup_cycle if seen[key]
+          return :destination_lookup_limit if seen.size >= @limit
+          seen[key] = true
+          nil
+        end
+
+        def destination_view(values)
+          return fact(reason: :malformed_destination) unless values.is_a?(Array)
+          fit, parameters = values[1], values.drop(2)
+          valid_parameters = parameters.all? { |value| value.nil? || (value.is_a?(Numeric) && value.finite?) }
+          valid = FIT_ARITY[fit] == parameters.size && valid_parameters
+          fact(fit: fit.is_a?(Symbol) ? fit : nil, parameters: valid_parameters ? parameters.freeze : [].freeze,
+            reason: valid ? nil : :invalid_destination_view)
+        end
+
+        def remote_destination(action)
+          file = remote_file(action[:F])
+          target = {action: :GoToR, file: file, page_index: nil, name: nil}.freeze
+          destination = remote_location(destination_value(action[:D]), target)
+          file ? destination : destination.with(kind: :unresolved, reason: :invalid_remote_target)
+        end
+
+        def destination_value(raw)
+          object = @document.wrap(@document.deref(raw))
+          object = @document.wrap(@document.deref(object[:D])) if object.value.is_a?(Hash)
+          object.value.is_a?(Array) ? object.to_a : object.value
+        end
+
+        def destination_name(value)
+          return unless named?(value)
+          text = value.to_s.encode(Encoding::UTF_8)
+          text.dup.freeze if text.valid_encoding? && !text.include?("\0")
+        rescue EncodingError
+          nil
+        end
+
+        def named?(value)
+          value.is_a?(String) || value.is_a?(Symbol)
+        end
+
+        def remote_file(raw)
+          file = @document.wrap(@document.deref(raw))
+          value = file.value.is_a?(Hash) ? file[:UF] || file[:F] : file.value
+          destination_name(value)
+        end
+
+        def remote_location(value, target)
+          return remote_name(value, target) if named?(value)
+          view = destination_view(value).with(remote_target: target)
+          return view if view.reason
+          index = value.first
+          return view.with(reason: :invalid_remote_page) unless index.is_a?(Integer) && index >= 0
+          view.with(kind: :remote, remote_target: target.merge(page_index: index).freeze)
+        end
+
+        def remote_name(value, target)
+          name = destination_name(value)
+          return fact(remote_target: target, reason: :invalid_destination_name) unless name
+          fact(kind: :remote, name: name, remote_target: target.merge(name: name).freeze)
+        end
+
+        def fact(kind: :unresolved, page_number: nil, name: nil, fit: nil, parameters: [].freeze, remote_target: nil, reason: nil)
+          Destination.new(kind: kind, page_number: page_number, name: name, fit: fit,
+            parameters: parameters, remote_target: remote_target, reason: reason)
+        end
+      end
+    end
+  end
+end