gento 0.1.1

data/lib/gento/html.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+require "strscan"
+require "cgi"
+require "json"
+
+module Gento
+  # A deliberately small HTML scanner.
+  #
+  # This is not a DOM parser and does not try to be one. It finds tags by name,
+  # reads their attributes, and grabs the raw text of non-void elements, which
+  # covers everything the adapters need (meta, link, script, img, title).
+  #
+  # Why not Nokogiri: it ships a native extension. Staying on pure Ruby keeps
+  # the gem usable from slim containers and from ruby.wasm, where compiling
+  # libxml is anywhere from painful to impossible.
+  class Html
+    VOID_ELEMENTS = %w[
+      area base br col embed hr img input link meta param source track wbr
+    ].freeze
+
+    attr_reader :source
+
+    def initialize(source)
+      # Defensive: adapters are the only caller in practice and Response has
+      # already normalized, but a caller handing us a raw body should not blow
+      # up mid-scan with an encoding error.
+      @source = Charset.normalize(source)
+      @tags = {}
+      @elements = {}
+    end
+
+    # Attribute hashes for every occurrence of `name`, in document order.
+    #
+    # Unlike #elements this never consumes element content, so it still finds
+    # tags nested inside another tag of the same name — which ordinary page
+    # markup is full of.
+    def tags(name)
+      @tags[name.to_s.downcase] ||= scan_tags(name.to_s)
+    end
+
+    # [attributes, inner_html] pairs for every occurrence of `name`.
+    #
+    # Only useful for elements that do not nest, such as <script> and <title>;
+    # for anything else use #tags.
+    def elements(name)
+      @elements[name.to_s.downcase] ||= scan_elements(name.to_s)
+    end
+
+    # Content of the first <meta> whose name/property/itemprop matches any key.
+    def meta(*keys)
+      meta_all(*keys).first
+    end
+
+    # Content of every matching <meta>, for properties that legitimately repeat.
+    def meta_all(*keys)
+      wanted = keys.flatten.map { |key| key.to_s.downcase }
+      tags("meta").filter_map { |attrs| meta_content(attrs, wanted) }
+    end
+
+    # href of the first <link> carrying the given rel token.
+    def link(rel)
+      wanted = rel.to_s.downcase
+      tags("link").each do |attrs|
+        tokens = attrs["rel"].to_s.downcase.split(/\s+/)
+        return attrs["href"] if tokens.include?(wanted)
+      end
+      nil
+    end
+
+    def title
+      inner = elements("title").first&.last
+      inner && decode_entities(inner.strip)
+    end
+
+    # Every parsed application/ld+json block, with @graph arrays flattened.
+    def json_ld
+      documents = elements("script").filter_map do |attrs, inner|
+        next unless attrs["type"].to_s.downcase.include?("ld+json")
+
+        parse_json(inner)
+      end
+      documents.flat_map { |doc| flatten_json_ld(doc) }
+    end
+
+    # Raw bodies of inline <script> tags, for sites that stash a JSON blob there.
+    def inline_scripts
+      elements("script").filter_map do |attrs, inner|
+        next if attrs.key?("src")
+
+        inner
+      end
+    end
+
+    # Any absolute URL appearing anywhere in the document, in document order,
+    # entity-decoded and deduplicated.
+    #
+    # Slide hosts put page images wherever suits them: Speaker Deck uses
+    # <a href>, Docswell a lazy-loading data attribute, Google Slides a CSS
+    # background inside a style attribute. Matching the URL shape across the
+    # raw document handles all three, and keeps working when a site moves its
+    # images from one element to another.
+    ABSOLUTE_URL = %r{https?://[^\s"'<>\\)]+}
+
+    def urls(pattern = //)
+      source.scan(ABSOLUTE_URL)
+            .map { |url| decode_entities(url) }
+            .grep(pattern)
+            .uniq
+    end
+
+    def decode_entities(text)
+      Entities.decode(text)
+    end
+
+    private
+
+    # A <meta> carries its key under any of three attributes depending on the
+    # vocabulary in use; an empty content is treated as absent.
+    def meta_content(attrs, wanted)
+      key = attrs["property"] || attrs["name"] || attrs["itemprop"]
+      return nil unless key && wanted.include?(key.downcase)
+
+      content = attrs["content"]
+      content unless content.nil? || content.empty?
+    end
+
+    def scan_tags(name)
+      results = []
+      scanner = StringScanner.new(@source)
+      opening = opening_pattern(name)
+
+      while scanner.skip_until(opening)
+        attributes, = scan_attributes(scanner)
+        results << attributes
+      end
+
+      results
+    end
+
+    def scan_elements(name)
+      results = []
+      scanner = StringScanner.new(@source)
+      opening = opening_pattern(name)
+      void = VOID_ELEMENTS.include?(name.downcase)
+
+      while scanner.skip_until(opening)
+        attributes, self_closing = scan_attributes(scanner)
+        inner = void || self_closing ? nil : scan_inner(scanner, name)
+        results << [attributes, inner]
+      end
+
+      results
+    end
+
+    def opening_pattern(name)
+      %r{<#{Regexp.escape(name)}(?=[\s>/])}i
+    end
+
+    # Consumes the attribute list and the closing `>` of an open tag.
+    def scan_attributes(scanner)
+      attributes = {}
+
+      loop do
+        scanner.skip(/\s+/)
+        return [attributes, true] if scanner.scan(%r{/\s*>})
+        return [attributes, false] if scanner.scan(">")
+        return [attributes, false] if scanner.eos?
+
+        name = scanner.scan(%r{[^\s=/>]+})
+        # Never stall: an unexpected byte here would otherwise loop forever.
+        next scanner.getch if name.nil?
+
+        scanner.skip(/\s*/)
+        attributes[name.downcase] = decode_entities(scan_attribute_value(scanner))
+      end
+    end
+
+    def scan_attribute_value(scanner)
+      return "" unless scanner.scan(/=\s*/)
+
+      if scanner.scan(/"([^"]*)"/) || scanner.scan(/'([^']*)'/)
+        scanner[1]
+      else
+        scanner.scan(/[^\s>]*/).to_s
+      end
+    end
+
+    # Returns the raw text up to the matching close tag.
+    #
+    # StringScanner#pos counts bytes while String#[] counts characters, so
+    # slicing has to go through byteslice or every multibyte document comes
+    # back mangled. Tag boundaries are ASCII, so byte slicing is safe here.
+    def scan_inner(scanner, name)
+      start = scanner.pos
+      closing = %r{</#{Regexp.escape(name)}\s*>}i
+
+      unless scanner.skip_until(closing)
+        scanner.terminate
+        return byteslice_from(start, @source.bytesize - start)
+      end
+
+      byteslice_from(start, scanner.pos - scanner.matched_size - start)
+    end
+
+    def byteslice_from(start, length)
+      slice = @source.byteslice(start, length).to_s
+      slice.valid_encoding? ? slice : slice.scrub
+    end
+
+    def parse_json(text)
+      JSON.parse(text.to_s)
+    rescue JSON::ParserError
+      nil
+    end
+
+    def flatten_json_ld(doc)
+      case doc
+      when Array then doc.flat_map { |entry| flatten_json_ld(entry) }
+      when Hash then doc.key?("@graph") ? flatten_json_ld(doc["@graph"]) + [doc] : [doc]
+      else []
+      end
+    end
+  end
+end

data/lib/gento/registry.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Gento
+  # Maps a URL to the adapter that claims it.
+  #
+  # Adding support for a new slide host is one class plus one `register` call;
+  # nothing else in the library needs to know the site exists.
+  class Registry
+    class << self
+      def default
+        @default ||= new(
+          [
+            Adapters::SpeakerDeck,
+            Adapters::SlideShare,
+            Adapters::Docswell,
+            Adapters::GoogleSlides
+          ]
+        )
+      end
+
+      attr_writer :default
+    end
+
+    def initialize(adapters = [])
+      @adapters = adapters.dup
+    end
+
+    def adapters
+      @adapters.dup.freeze
+    end
+
+    def register(adapter)
+      @adapters.unshift(adapter)
+      self
+    end
+
+    def find(url)
+      @adapters.find { |adapter| adapter.handles?(url) }
+    end
+
+    def supports?(url)
+      !find(url).nil?
+    end
+
+    def providers
+      @adapters.map(&:provider)
+    end
+  end
+end

data/lib/gento/response.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gento
+  # A minimal HTTP response, so adapters never touch a fetcher-specific object.
+  class Response
+    attr_reader :status, :headers, :body, :url
+
+    def initialize(status:, body:, url:, headers: {})
+      @status = Integer(status)
+      @url = url.to_s
+      @headers = headers.transform_keys { |key| key.to_s.downcase }.freeze
+      @body = Charset.normalize(body, content_type: @headers["content-type"])
+      freeze
+    end
+
+    def success?
+      status.between?(200, 299)
+    end
+
+    def content_type
+      headers["content-type"]
+    end
+
+    def json
+      JSON.parse(body)
+    rescue JSON::ParserError => e
+      raise ExtractionError, "expected JSON from #{url}: #{e.message}"
+    end
+
+    def html
+      Html.new(body)
+    end
+  end
+end

data/lib/gento/robots.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Gento
+  # A parsed robots.txt, following RFC 9309.
+  #
+  # The one rule worth implementing carefully is precedence: **the longest
+  # matching pattern wins**, not the first one. Google's robots.txt ends with
+  #
+  #   Allow: /presentation
+  #   Disallow: /
+  #
+  # and a first-match reading would conclude that every Google Slides deck is
+  # off limits, which is the opposite of what that file says.
+  class Robots
+    # A single Allow or Disallow line.
+    class Rule
+      attr_reader :pattern
+
+      def initialize(pattern, allow:)
+        @pattern = pattern
+        @allow = allow
+        @regexp = self.class.compile(pattern)
+        freeze
+      end
+
+      def allow?
+        @allow
+      end
+
+      def covers?(path)
+        @regexp.match?(path)
+      end
+
+      # RFC 9309 orders rules by the length of the pattern, wildcards included.
+      def specificity
+        pattern.length
+      end
+
+      # `*` stands for any run of characters and `$` anchors the end; every
+      # other character is literal.
+      def self.compile(pattern)
+        anchored = pattern.end_with?("$")
+        body = anchored ? pattern[0..-2] : pattern
+        source = body.split("*", -1).map { |part| Regexp.escape(part) }.join(".*")
+        source += "\\z" if anchored
+
+        Regexp.new("\\A#{source}")
+      end
+    end
+
+    Group = Struct.new(:agents, :rules, :crawl_delay)
+
+    DIRECTIVES = %w[user-agent allow disallow crawl-delay].freeze
+    # A product token runs to the first slash or space: the "gento" of
+    # "gento/0.1.0 (+https://...)".
+    PRODUCT_TOKEN = %r{\A[^\s/]+}
+
+    attr_reader :groups, :reason
+
+    def initialize(groups = [], reason: nil)
+      @groups = groups.freeze
+      @reason = reason
+      freeze
+    end
+
+    class << self
+      def parse(body)
+        new(Parser.new(body).groups)
+      end
+
+      # No robots.txt to obey — the host said so with a 404, or answered with
+      # something that is not a robots.txt at all.
+      def allow_all
+        new
+      end
+
+      # RFC 9309 §2.3.1.4: a robots.txt that cannot be read is not permission
+      # to crawl. `reason` explains which failure it was.
+      def disallow_all(reason: nil)
+        new([Group.new(["*"], [Rule.new("/", allow: false)], nil)], reason: reason)
+      end
+
+      def product_token(user_agent)
+        user_agent.to_s[PRODUCT_TOKEN].to_s.downcase
+      end
+    end
+
+    def allowed?(path, agent: nil)
+      group = group_for(agent)
+      return true unless group
+
+      matching = group.rules.select { |rule| rule.covers?(path) }
+      return true if matching.empty?
+
+      # Longest pattern wins; an Allow and a Disallow of equal length are a
+      # tie the site did not resolve, and RFC 9309 gives those to Allow.
+      matching.max_by { |rule| [rule.specificity, rule.allow? ? 1 : 0] }.allow?
+    end
+
+    def crawl_delay(agent: nil)
+      group_for(agent)&.crawl_delay
+    end
+
+    def empty?
+      groups.empty?
+    end
+
+    private
+
+    # A crawler obeys exactly one group: the one naming it most specifically,
+    # falling back to `*`.
+    def group_for(agent)
+      token = self.class.product_token(agent)
+
+      groups.filter_map { |group| [group, specificity(group, token)] if applies?(group, token) }
+            .max_by(&:last)
+            &.first
+    end
+
+    def applies?(group, token)
+      group.agents.any? { |name| name == "*" || (!token.empty? && token.start_with?(name)) }
+    end
+
+    def specificity(group, token)
+      group.agents.filter_map { |name| name.length if name != "*" && token.start_with?(name) }
+           .max || 0
+    end
+
+    # robots.txt is a line-oriented `key: value` format. Anything unrecognised
+    # is skipped rather than treated as an error, which is also what keeps a
+    # page served in place of a robots.txt from parsing into rules.
+    class Parser
+      attr_reader :groups
+
+      def initialize(body)
+        @groups = []
+        @current = nil
+        @reading_agents = false
+
+        each_directive(body) { |key, value| apply(key, value) }
+      end
+
+      private
+
+      def apply(key, value)
+        case key
+        when "user-agent" then add_agent(value)
+        when "allow", "disallow" then add_rule(value, allow: key == "allow")
+        when "crawl-delay" then record_crawl_delay(value)
+        end
+      end
+
+      # Consecutive User-agent lines share one group; a rule line ends the
+      # list, so the next User-agent starts a new group.
+      def add_agent(value)
+        unless @reading_agents && @current
+          @current = Group.new([], [], nil)
+          @groups << @current
+        end
+
+        @current.agents << value.downcase
+        @reading_agents = true
+      end
+
+      def add_rule(value, allow:)
+        @reading_agents = false
+        # "Disallow:" with nothing after it lifts the restriction rather than
+        # imposing one, so there is no rule to record.
+        return if @current.nil? || value.empty?
+
+        @current.rules << Rule.new(value, allow: allow)
+      end
+
+      def record_crawl_delay(value)
+        @reading_agents = false
+        return if @current.nil?
+
+        @current.crawl_delay = Float(value)
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def each_directive(body)
+        body.to_s.each_line do |line|
+          line = line.sub(/#.*/, "").strip
+          key, separator, value = line.partition(":")
+          next if separator.empty?
+
+          key = key.strip.downcase
+          yield key, value.strip if DIRECTIVES.include?(key)
+        end
+      end
+    end
+  end
+end

data/lib/gento/robots_fetcher.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "uri"
+
+module Gento
+  # A Fetcher that consults robots.txt before letting a request through.
+  #
+  # It wraps another fetcher rather than living inside one, so it applies to
+  # every request an adapter makes — the deck page, the oEmbed endpoint, the
+  # embed view — and works just as well over an injected fetcher as over the
+  # net/http one.
+  #
+  # On by default. `Gento.fetch(url, robots: false)` turns it off; the
+  # caller then owns whatever that implies.
+  #
+  # One gap: the wrapped fetcher follows redirects itself, so a redirect into
+  # a disallowed path is not caught. Checking that needs a per-hop callback in
+  # the fetcher, which is not worth the interface it would add.
+  class RobotsFetcher < Fetcher
+    ROBOTS_PATH = "/robots.txt"
+    # Speaker Deck serves its robots.txt through the site's HTML layout unless
+    # this header asks for plain text, so it is load-bearing rather than
+    # politeness. The body is parsed whatever content type comes back: a page
+    # that is not a robots.txt yields no directives anyway, and refusing on
+    # content type alone would throw away real rules from a host that answers
+    # in HTML.
+    PLAIN_TEXT = { "accept" => "text/plain" }.freeze
+    DEFAULT_PORTS = { "http" => 80, "https" => 443 }.freeze
+
+    attr_reader :fetcher, :user_agent
+
+    def initialize(fetcher, user_agent: nil)
+      @fetcher = fetcher
+      @user_agent = user_agent || inherited_user_agent(fetcher)
+      @robots = {}
+      super()
+    end
+
+    def get(url, headers: {})
+      uri = coerce_uri(url)
+      guard!(uri) if uri
+
+      fetcher.get(url, headers: headers)
+    end
+
+    # The Crawl-delay the host asks for, in seconds, or nil.
+    #
+    # Not enforced here: one fetch is a handful of requests, and sleeping
+    # inside a fetcher would surprise a caller who already paces its own work.
+    # Exposed so a caller doing more than one deck can honour it.
+    def crawl_delay(url)
+      uri = coerce_uri(url)
+      return nil unless uri
+
+      robots_for(uri).crawl_delay(agent: user_agent)
+    end
+
+    def robots_for(uri)
+      @robots[origin(uri)] ||= load_robots(origin(uri))
+    end
+
+    private
+
+    def guard!(uri)
+      robots = robots_for(uri)
+      path = request_path(uri)
+      return if robots.allowed?(path, agent: user_agent)
+
+      raise RobotsDisallowedError.new(refusal(uri, path, robots), url: uri.to_s)
+    end
+
+    def refusal(uri, path, robots)
+      cause = robots.reason || "#{origin(uri)}#{ROBOTS_PATH} disallows #{path}"
+
+      "#{cause} for #{user_agent_label}. " \
+        "Pass robots: false to fetch it anyway, and take responsibility for doing so."
+    end
+
+    def user_agent_label
+      user_agent.to_s.empty? ? "this client" : user_agent.inspect
+    end
+
+    def load_robots(origin)
+      Robots.parse(fetcher.get("#{origin}#{ROBOTS_PATH}", headers: PLAIN_TEXT).body)
+    rescue ResponseError => e
+      absent?(e.status) ? Robots.allow_all : unreachable(origin, "responded #{e.status}")
+    rescue FetchError => e
+      unreachable(origin, e.message)
+    end
+
+    # 4xx is the host saying there is no robots.txt, which RFC 9309 reads as
+    # "help yourself". 429 is it saying to come back later, which is not the
+    # same thing.
+    def absent?(status)
+      status.to_i.between?(400, 499) && status.to_i != 429
+    end
+
+    def unreachable(origin, detail)
+      Robots.disallow_all(reason: "could not read #{origin}#{ROBOTS_PATH} (#{detail}), " \
+                                  "so there is no permission to rely on")
+    end
+
+    def origin(uri)
+      port = ":#{uri.port}" unless uri.port == DEFAULT_PORTS[uri.scheme]
+
+      "#{uri.scheme}://#{uri.host}#{port}"
+    end
+
+    # Rules match against the path and query together.
+    def request_path(uri)
+      path = uri.path.to_s.empty? ? "/" : uri.path
+      uri.query ? "#{path}?#{uri.query}" : path
+    end
+
+    def coerce_uri(url)
+      uri = url.is_a?(URI) ? url : URI.parse(url.to_s)
+      uri.is_a?(URI::HTTP) ? uri : nil
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def inherited_user_agent(fetcher)
+      fetcher.user_agent if fetcher.respond_to?(:user_agent)
+    end
+  end
+end

data/lib/gento/slide.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Gento
+  # A single page of a deck.
+  #
+  # `url` always points at the original host. We deliberately never re-host or
+  # proxy slide images: this library reports where the images are, and leaves
+  # serving them to the site that owns them.
+  class Slide
+    attr_reader :number, :url, :width, :height, :thumbnail_url
+
+    def initialize(number:, url:, width: nil, height: nil, thumbnail_url: nil)
+      @number = Integer(number)
+      @url = url.to_s
+      @width = width && Integer(width)
+      @height = height && Integer(height)
+      @thumbnail_url = thumbnail_url&.to_s
+      freeze
+    end
+
+    def to_h
+      {
+        number: number,
+        url: url,
+        width: width,
+        height: height,
+        thumbnail_url: thumbnail_url
+      }.compact
+    end
+
+    def ==(other)
+      other.is_a?(Slide) && other.to_h == to_h
+    end
+    alias eql? ==
+
+    def hash
+      to_h.hash
+    end
+  end
+end

data/lib/gento/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Gento
+  VERSION = "0.1.1"
+end