substrate 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7b3e7c25e9af2d9e3b590493a0b749c1413c8fc4542074a891f0879fb308dd2
-  data.tar.gz: e88153c370e8f419a810c7014b9b5e15790741811ed40874ac5f3517dba0b8c3
+  metadata.gz: f70ea3ee7d06a49e41215d92a1b04db9173cbf357cbeccbd9bb85adc281b2e45
+  data.tar.gz: b86bcb7ff7b2f9eab771a2e3419b1ab0cf42a08bb4ae6d4bd906d2cee0ea0c5f
 SHA512:
-  metadata.gz: 4200859c9d863f6d18692d9c92d97208cab9b07b17aa9d73f791724917752cc4261709f4fc49216ad2c0a71833d6fec9917c6f0e94972c7287f6f03046888b61
-  data.tar.gz: 4fb43377d34974ce6f02caaef6a2437e34b3a2d21948776f3ca1ba6d5955250365c576c3e9f7776750c95394c35f0806bea45aaa6449198ebf908d101a48ee75
+  metadata.gz: 2d2687f91dbe8ca128de93e975de5e0a25359e8e8f1d7399b6092721b553ac26985eeff2d2963a7112f70bf11f98fdbcfff8f30e4462cb9fc9b3375349e6cf37
+  data.tar.gz: e303be1550bd6d5f98635689316e9be8e17d88657e196d614133f84aad0674745db45ffb0f1702dfc22ac3f5798e923113f612151fe98a71bfbce8ba602b0542

data/lib/substrate/classify.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Substrate
+  class Classify < Command
+    class UnsupportedSource < StandardError; end
+    class ResolutionError < StandardError; end
+
+    EXTENSION_MAP = {
+      ".txt" => :text_file,
+      ".md" => :text_file,
+      ".rst" => :text_file,
+      ".pdf" => :pdf_file,
+      ".mp3" => :audio_file
+    }.freeze
+
+    def initialize(input:)
+      @input = input
+    end
+
+    def call
+      success(payload: classified)
+    end
+
+    private
+
+    attr_reader :input
+
+    def classified
+      return RawText.new(text: input) unless resolved_path || url?
+
+      return Sources::Url.new(input) if url?
+
+      case file_kind
+      when :pdf_file then Sources::Pdf.new(resolved_path)
+      when :text_file then Sources::Text.new(resolved_path)
+      when :audio_file then Sources::Audio.new(resolved_path)
+      end
+    end
+
+    def url?
+      uri = URI.parse(input)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def file_kind
+      ext = File.extname(resolved_path).downcase
+      EXTENSION_MAP.fetch(ext) do
+        raise UnsupportedSource, "Unsupported file type: #{ext}"
+      end
+    end
+
+    def resolved_path
+      @resolved_path ||= begin
+        result = Detection::PathResolver.call(raw: input)
+        raise ResolutionError, result.error.message unless result.success?
+
+        result.payload
+      end
+    end
+  end
+end

data/lib/substrate/cli/availability.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Cli
+    class Availability
+      def self.call(executable)
+        system("which", executable, out: File::NULL, err: File::NULL)
+      end
+    end
+  end
+end

data/lib/substrate/cli/error.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Cli
+    class Error < StandardError; end
+  end
+end

data/lib/substrate/cli/runner.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Substrate
+  module Cli
+    class Runner < Command
+      ExecutionError = Class.new(Error)
+
+      def initialize(argv:)
+        @argv = argv
+      end
+
+      def call
+        stdout, stderr, status = Open3.capture3(*argv)
+
+        raise ExecutionError, stderr unless status.success?
+
+        success(payload: stdout)
+      rescue ExecutionError
+        raise
+      rescue => e
+        raise ExecutionError, e.message
+      end
+
+      private
+
+      attr_reader :argv
+    end
+  end
+end

data/lib/substrate/command.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Substrate
+  Result = Data.define(:payload, :error) do
+    def success? = error.nil?
+    def failure? = !error.nil?
+  end
+
+  class Command
+    def self.call(...)
+      new(...).call
+    rescue => e
+      Result.new(payload: nil, error: e)
+    end
+
+    def success(payload:)
+      Result.new(payload: payload, error: nil)
+    end
+
+    def failure(error)
+      Result.new(payload: nil, error: error)
+    end
+  end
+end

data/lib/substrate/configuration.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Configuration
+    class << self
+      def deepgram_api_key
+        @deepgram_api_key ||= ENV.fetch("DEEPGRAM_API_KEY", nil)
+      end
+
+      attr_writer :deepgram_api_key
+
+      def configure
+        yield self
+      end
+
+      def reset!
+        @deepgram_api_key = nil
+      end
+    end
+  end
+end

data/lib/substrate/detection/path_resolver.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Substrate
+  module Detection
+    class PathResolver < Command
+      def initialize(raw:)
+        @raw = raw
+      end
+
+      def call
+        path = candidates.find { |p| p.file? }
+        success(payload: path&.to_s)
+      end
+
+      private
+
+      attr_reader :raw
+
+      def candidates
+        [raw, shell_unescape].compact.uniq.map { |s| Pathname.new(s).expand_path }
+      end
+
+      def shell_unescape
+        parts = Shellwords.shellsplit(raw)
+        parts.first if parts.length == 1
+      rescue ArgumentError
+        nil
+      end
+    end
+  end
+end

data/lib/substrate/distill.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Substrate
+  class Distill < Command
+    def initialize(content:)
+      @content = content
+    end
+
+    def call
+      case content
+      in RawText(text:)
+        Handlers::RawText.call(text: text)
+      in TextContent(source_ref:, text:)
+        Handlers::TextFile.call(text: text, source_ref: source_ref)
+      in ArticleContent(source_ref:, text:, title:, metadata:)
+        Handlers::Article.call(content: content)
+      in PageCollection(source_ref:, pages:)
+        document = Mediums::Pdf::Document.new(source_ref.locator, source_ref: source_ref)
+        Handlers::Pdf.call(document: document, pages: pages)
+      in Transcripts::Content(source_ref:, segments:, metadata:)
+        Handlers::Transcript.call(content: content)
+      else
+        failure(ArgumentError.new("Unsupported content type: #{content.class}"))
+      end
+    end
+
+    private
+
+    attr_reader :content
+  end
+end

data/lib/substrate/extract_pdf.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "hexapdf"
+require "stringio"
+
+module Substrate
+  class ExtractPdf < Command
+    def initialize(bytes:, source_ref:, limits: Mediums::Pdf::Limits.new, page_separator: "\n\n")
+      @bytes = bytes
+      @source_ref = source_ref
+      @limits = limits
+      @page_separator = page_separator
+    end
+
+    def call
+      validate_input
+      validate_page_separator
+      extract_document
+    end
+
+    private
+
+    def validate_input
+      raise ArgumentError, "bytes must be a String" unless @bytes.is_a?(String)
+      @limits.check!(:input_bytes, @bytes.bytesize)
+    end
+
+    def validate_page_separator
+      raise ArgumentError, "page_separator must be a String" unless @page_separator.is_a?(String)
+      @page_separator = @page_separator.encode(Encoding::UTF_8).dup.freeze
+      raise ArgumentError, "page_separator must contain valid UTF-8" unless @page_separator.valid_encoding?
+    rescue EncodingError => error
+      raise ArgumentError, "page_separator must contain valid UTF-8", cause: error
+    end
+
+    def extract_document
+      StringIO.open(@bytes) do |io|
+        document = HexaPDF::Document.new(io: io, config: {"document.auto_decrypt" => false, "filter.map" => {BrotliDecode: nil}})
+        raise Mediums::Pdf::EncryptedPdf, "Encrypted PDFs are not supported" if document.encrypted?
+        objects = page_objects(document)
+        pages = extract_pages(objects)
+        outline = Mediums::Pdf::OutlineFacts.new(document, objects, @limits).call
+        publish_content(pages, outline)
+      end
+    rescue HexaPDF::Error => error
+      raise Mediums::Pdf::InvalidText, "PDF Unicode mapping is invalid", cause: error if error.cause.is_a?(EncodingError)
+      raise Mediums::Pdf::InvalidPdf, "PDF parsing or page extraction failed", cause: error
+    end
+
+    def page_objects(document)
+      root = document.catalog[:Pages]
+      raise Mediums::Pdf::InvalidPdf, "PDF page tree is missing" unless root.is_a?(HexaPDF::Dictionary)
+      count = root[:Count]
+      raise Mediums::Pdf::InvalidPdf, "PDF page count is invalid" unless count.is_a?(Integer) && count >= 0
+      @limits.check!(:pages, count)
+      pending = [[root, nil]]
+      seen = {}
+      [].tap do |pages|
+        until pending.empty?
+          candidate, parent = pending.pop
+          node = document.deref(candidate)
+          raise Mediums::Pdf::InvalidPdf, "PDF page tree node is invalid" unless node.is_a?(HexaPDF::Dictionary)
+          raise Mediums::Pdf::InvalidPdf, "PDF page ancestry is inconsistent" unless node[:Parent] == parent
+          identity = node.indirect? ? [node.oid, node.gen] : node.data.object_id
+          raise Mediums::Pdf::InvalidPdf, "PDF page tree contains a cycle or repeated node" if seen[identity]
+          seen[identity] = true
+          if node[:Type] == :Page
+            @limits.check!(:pages, pages.size + 1)
+            pages << document.wrap(node, type: :Page)
+          elsif node[:Type] == :Pages
+            kids = node[:Kids]
+            raise Mediums::Pdf::InvalidPdf, "PDF page tree children are invalid" unless kids.is_a?(HexaPDF::PDFArray)
+            pending.concat(kids.to_a.reverse.map { |child| [child, node] })
+          else
+            raise Mediums::Pdf::InvalidPdf, "PDF page tree type is invalid"
+          end
+        end
+      end
+    end
+
+    def extract_pages(objects)
+      @text = +""
+      objects.map.with_index do |page, index|
+        @limits.check!(:pages, index + 1)
+        text = page.extract_text.encode(Encoding::UTF_8)
+        raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8 or NUL" unless text.valid_encoding? && !text.include?("\0")
+        append_page_text(text, index)
+        Mediums::Pdf::Page.new(number: index + 1, text: text.freeze, crop_box: crop_box(page))
+      end.freeze.tap do |pages|
+        raise Mediums::Pdf::NoSelectableText, "PDF contains no selectable text" if pages.all? { |page| page.text.match?(/\A[[:space:]]*\z/) }
+      end
+    rescue EncodingError => error
+      raise Mediums::Pdf::InvalidText, "PDF text contains invalid UTF-8", cause: error
+    end
+
+    def append_page_text(text, index)
+      separator = index.zero? ? "" : @page_separator
+      @limits.check!(:text_bytes, @text.bytesize + separator.bytesize + text.bytesize)
+      @text << separator << text
+    end
+
+    def crop_box(page)
+      media, crop = page_rectangles(page)
+      return unless usable_box?(media) && usable_box?(crop)
+      box = Mediums::Pdf::Box.new(left: [media.left, crop.left].max,
+        bottom: [media.bottom, crop.bottom].max, right: [media.right, crop.right].min,
+        top: [media.top, crop.top].min)
+      box if box.right > box.left && box.top > box.bottom
+    end
+
+    def page_rectangles(page)
+      media = page[:MediaBox]
+      crop = page[:CropBox]
+      [media, crop.nil? ? media : crop]
+    rescue HexaPDF::Error
+      [nil, nil]
+    end
+
+    def usable_box?(rectangle)
+      rectangle.is_a?(HexaPDF::Rectangle) &&
+        rectangle.to_a.all? { |value| value.is_a?(Numeric) && value.finite? } &&
+        rectangle.right > rectangle.left && rectangle.top > rectangle.bottom
+    end
+
+    def publish_content(pages, outline)
+      metadata = {extractor: "HexaPDF", extractor_version: HexaPDF::VERSION.dup.freeze,
+                  input_bytes: @bytes.bytesize, page_count: pages.size,
+                  text_bytes: @text.bytesize}.freeze
+      success(payload: Mediums::Pdf::Content.new(pages: pages, source_ref: @source_ref,
+        metadata: metadata, outline: outline, text: @text.freeze))
+    end
+  end
+end

data/lib/substrate/handlers/article.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Article < Command
+      def initialize(content:)
+        @content = content
+      end
+
+      def call
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :content
+
+      def extraction
+        Extraction.new(
+          text: content.text,
+          source: source,
+          metadata: extraction_metadata
+        )
+      end
+
+      def source
+        Source.new(
+          kind: :article,
+          locator: content.source_ref.locator,
+          metadata: content.metadata
+        )
+      end
+
+      def extraction_metadata
+        {title: content.title}.compact
+      end
+    end
+  end
+end

data/lib/substrate/handlers/pdf.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Pdf < Command
+      def initialize(document:, pages: nil)
+        @document = document
+        @pages = pages
+      end
+
+      def call
+        range = pages || (1..document.page_count)
+        text = document.pages_text(range)
+        source = Source.new(kind: :pdf_file, locator: nil, metadata: {
+          page_count: document.page_count
+        })
+        extraction = Extraction.new(text: text, source: source, metadata: {
+          pages_extracted: range
+        })
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :document, :pages
+    end
+  end
+end

data/lib/substrate/handlers/raw_text.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class RawText < Command
+      def initialize(text:)
+        @text = text
+      end
+
+      def call
+        source = Source.new(kind: :raw_text, locator: nil, metadata: {})
+        extraction = Extraction.new(text: text, source: source, metadata: {})
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :text
+    end
+  end
+end

data/lib/substrate/handlers/text_file.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class TextFile < Command
+      def initialize(text:, source_ref:)
+        @text = text
+        @source_ref = source_ref
+      end
+
+      def call
+        source = Source.new(kind: :text_file, locator: source_ref.locator, metadata: {
+          line_count: text.count("\n"),
+          file_size: File.size(source_ref.locator)
+        })
+        extraction = Extraction.new(text: text, source: source, metadata: {})
+
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :text, :source_ref
+    end
+  end
+end

data/lib/substrate/handlers/transcript.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Handlers
+    class Transcript < Command
+      def initialize(content:)
+        @content = content
+      end
+
+      def call
+        success(payload: extraction)
+      end
+
+      private
+
+      attr_reader :content
+
+      def extraction
+        Extraction.new(
+          text: full_text,
+          source: source,
+          metadata: extraction_metadata
+        )
+      end
+
+      def full_text
+        content.segments.map(&:text).join("\n\n")
+      end
+
+      def source
+        Source.new(
+          kind: :audio,
+          locator: content.source_ref.locator,
+          metadata: content.metadata
+        )
+      end
+
+      def extraction_metadata
+        {
+          segment_count: content.segments.count,
+          duration_seconds: content.metadata.duration_seconds,
+          language: content.metadata.language,
+          diarized: content.metadata.diarized
+        }.compact
+      end
+    end
+  end
+end

data/lib/substrate/mediums/pdf/box.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Box = Data.define(:left, :bottom, :right, :top)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/chapter.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Chapter = Data.define(:source_ref, :id, :title, :pages)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/content.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Content = Data.define(:pages, :source_ref, :metadata, :outline, :text)
+    end
+  end
+end

data/lib/substrate/mediums/pdf/destination.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Substrate
+  module Mediums
+    module Pdf
+      Destination = Data.define(:kind, :page_number, :name, :fit, :parameters, :remote_target, :reason)
+    end
+  end
+end