gento 0.1.0

data/lib/gento/charset.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Gento
+  # Turns a raw response body into a UTF-8 string we can safely scan.
+  #
+  # net/http hands back ASCII-8BIT, and Japanese slide decks are the whole
+  # point of this library, so guessing the charset properly is not optional.
+  module Charset
+    DEFAULT = "UTF-8"
+    IN_CONTENT_TYPE = /charset\s*=\s*"?([\w-]+)"?/i
+    IN_META = /<meta[^>]+charset\s*=\s*["']?([\w-]+)/i
+    # A charset declaration must appear early; do not scan a whole document.
+    META_SCAN_BYTES = 4096
+
+    module_function
+
+    def normalize(body, content_type: nil)
+      body = body.to_s
+      name = from_content_type(content_type) || from_meta(body) || DEFAULT
+      transcode(body, name)
+    end
+
+    def from_content_type(content_type)
+      content_type && content_type[IN_CONTENT_TYPE, 1]
+    end
+
+    def from_meta(body)
+      head = body.byteslice(0, META_SCAN_BYTES).to_s.dup.force_encoding(::Encoding::ASCII_8BIT)
+      head[IN_META, 1]
+    end
+
+    def transcode(body, name)
+      encoding = ::Encoding.find(name)
+      decoded = body.dup.force_encoding(encoding)
+      decoded = decoded.encode(::Encoding::UTF_8, invalid: :replace, undef: :replace) unless utf8?(encoding)
+      decoded.valid_encoding? ? decoded : decoded.scrub
+    rescue ArgumentError, ::Encoding::ConverterNotFoundError, ::Encoding::UndefinedConversionError
+      body.dup.force_encoding(::Encoding::UTF_8).scrub
+    end
+
+    def utf8?(encoding)
+      encoding == ::Encoding::UTF_8
+    end
+  end
+end

data/lib/gento/cli.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+
+module Gento
+  # `gento <URL>` — prints the deck as JSON, or one image URL per line.
+  class CLI
+    EXIT_SUCCESS = 0
+    EXIT_FAILURE = 1
+    EXIT_USAGE = 2
+
+    def self.start(argv, stdout: $stdout, stderr: $stderr)
+      new(stdout: stdout, stderr: stderr).run(argv)
+    end
+
+    def initialize(stdout: $stdout, stderr: $stderr)
+      @stdout = stdout
+      @stderr = stderr
+      @options = { format: :json, pretty: true, robots: true }
+    end
+
+    def run(argv)
+      urls = parser.parse(argv.dup)
+
+      if urls.empty?
+        @stderr.puts parser.help
+        return EXIT_USAGE
+      end
+
+      fetch_all(urls)
+    rescue OptionParser::ParseError => e
+      @stderr.puts "gento: #{e.message}"
+      EXIT_USAGE
+    end
+
+    private
+
+    def fetch_all(urls)
+      client = Client.new(fetcher: build_fetcher, robots: @options[:robots])
+      status = EXIT_SUCCESS
+
+      urls.each do |url|
+        emit(client.fetch(url))
+      rescue Error => e
+        @stderr.puts "gento: #{url}: #{e.message}"
+        status = EXIT_FAILURE
+      end
+
+      status
+    end
+
+    def emit(deck)
+      case @options[:format]
+      when :urls then deck.slides.each { |slide| @stdout.puts slide.url }
+      else @stdout.puts serialize(deck)
+      end
+    end
+
+    def serialize(deck)
+      @options[:pretty] ? JSON.pretty_generate(deck.to_h) : deck.to_h.to_json
+    end
+
+    def build_fetcher
+      NetHttpFetcher.new(**{
+        user_agent: @options[:user_agent],
+        read_timeout: @options[:timeout]
+      }.compact)
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: gento [options] URL [URL...]"
+        opts.separator ""
+        opts.separator "Supported providers: #{Gento.providers.join(', ')}"
+        opts.separator ""
+
+        define_output_options(opts)
+        define_request_options(opts)
+        define_meta_options(opts)
+      end
+    end
+
+    def define_output_options(opts)
+      opts.on("-f", "--format FORMAT", %i[json urls],
+              "Output format: json (default) or urls") do |format|
+        @options[:format] = format
+      end
+
+      opts.on("--[no-]pretty", "Pretty-print JSON output (default: on)") do |pretty|
+        @options[:pretty] = pretty
+      end
+    end
+
+    def define_request_options(opts)
+      opts.on("-t", "--timeout SECONDS", Integer, "Read timeout per request") do |timeout|
+        @options[:timeout] = timeout
+      end
+
+      opts.on("-A", "--user-agent STRING", "User-Agent to send") do |agent|
+        @options[:user_agent] = agent
+      end
+
+      opts.on("--[no-]robots", "Obey the host's robots.txt (default: on)") do |robots|
+        @options[:robots] = robots
+      end
+    end
+
+    def define_meta_options(opts)
+      opts.on("-h", "--help", "Show this message") do
+        @stdout.puts opts
+        exit EXIT_SUCCESS
+      end
+
+      opts.on("-v", "--version", "Show the version") do
+        @stdout.puts Gento::VERSION
+        exit EXIT_SUCCESS
+      end
+    end
+  end
+end

data/lib/gento/client.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gento
+  # The entry point: give it a slide URL, get a Deck back.
+  #
+  #   client = Gento::Client.new
+  #   deck = client.fetch("https://speakerdeck.com/user/talk")
+  #   deck.slides.map(&:url)
+  #
+  # Both collaborators are injectable: swap the fetcher to run on a host with
+  # its own HTTP stack, swap the registry to limit or extend the supported
+  # sites.
+  #
+  # Every request goes through the host's robots.txt unless robots: false is
+  # passed. This holds for an injected fetcher too — the point of the check is
+  # the request, not who makes it.
+  class Client
+    attr_reader :fetcher, :registry
+
+    def initialize(fetcher: nil, registry: nil, robots: true)
+      @registry = registry || Registry.default
+      @fetcher = wrap(fetcher || NetHttpFetcher.new, robots: robots)
+    end
+
+    def robots?
+      fetcher.is_a?(RobotsFetcher)
+    end
+
+    # @raise [UnsupportedURLError] when no adapter claims the URL
+    # @raise [RobotsDisallowedError] when robots.txt does not allow the fetch
+    # @raise [FetchError] when the remote host cannot be reached
+    # @raise [ExtractionError] when the page held no slides
+    # @return [Deck]
+    def fetch(url)
+      adapter = registry.find(url)
+      raise UnsupportedURLError, url unless adapter
+
+      adapter.new(fetcher: fetcher).fetch(url.to_s)
+    end
+
+    def supports?(url)
+      registry.supports?(url)
+    end
+
+    private
+
+    # A fetcher that already checks robots.txt is left alone, so passing one
+    # in explicitly does not stack two checks.
+    def wrap(fetcher, robots:)
+      return fetcher if !robots || fetcher.is_a?(RobotsFetcher)
+
+      RobotsFetcher.new(fetcher)
+    end
+  end
+end

data/lib/gento/deck.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Gento
+  # The result of fetching one slide URL: metadata plus the ordered pages.
+  class Deck
+    attr_reader :provider, :source_url, :title, :author, :description,
+                :published_at, :slides
+
+    def initialize(provider:, source_url:, slides:, title: nil, author: nil,
+                   description: nil, published_at: nil)
+      @provider = provider.to_s
+      @source_url = source_url.to_s
+      @title = normalize(title)
+      @author = normalize(author)
+      @description = normalize(description)
+      @published_at = normalize(published_at)
+      @slides = slides.sort_by(&:number).freeze
+      freeze
+    end
+
+    def page_count
+      slides.size
+    end
+
+    def to_h
+      {
+        provider: provider,
+        source_url: source_url,
+        title: title,
+        author: author,
+        description: description,
+        published_at: published_at,
+        page_count: page_count,
+        slides: slides.map(&:to_h)
+      }.compact
+    end
+
+    def to_json(*args)
+      to_h.to_json(*args)
+    end
+
+    private
+
+    # Slide titles are typed into a design tool, so they arrive carrying the
+    # author's own line breaks — vertical tabs, in Docswell's case. Collapsing
+    # them here means every adapter gets it right without remembering to.
+    def normalize(text)
+      return nil if text.nil?
+
+      collapsed = text.to_s.gsub(/[[:space:][:cntrl:]]+/, " ").strip
+      collapsed.empty? ? nil : collapsed
+    end
+  end
+end

data/lib/gento/entities.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+module Gento
+  # HTML entity decoding.
+  #
+  # CGI.unescapeHTML handles the five predefined entities and numeric
+  # references, which leaves the typographic ones that real slide titles and
+  # descriptions are full of. Rather than carry the whole HTML5 entity table,
+  # translate the handful that actually show up.
+  module Entities
+    NAMED = {
+      "hellip" => "\u2026", "mdash" => "\u2014", "ndash" => "\u2013", "nbsp" => "\u00A0",
+      "lsquo" => "\u2018", "rsquo" => "\u2019", "ldquo" => "\u201C", "rdquo" => "\u201D",
+      "laquo" => "\u00AB", "raquo" => "\u00BB", "bull" => "\u2022", "middot" => "\u00B7",
+      "times" => "\u00D7", "deg" => "\u00B0", "copy" => "\u00A9", "reg" => "\u00AE",
+      "trade" => "\u2122"
+    }.freeze
+
+    PATTERN = /&(#{Regexp.union(NAMED.keys)});/
+
+    module_function
+
+    def decode(text)
+      CGI.unescapeHTML(text.to_s).gsub(PATTERN) { NAMED.fetch(Regexp.last_match(1)) }
+    end
+  end
+end

data/lib/gento/errors.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gento
+  # Base class for every error raised by this library. Callers that just want
+  # "did it work?" can rescue this one class.
+  class Error < StandardError; end
+
+  # The given URL is not a URL we know how to handle.
+  class UnsupportedURLError < Error
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      super("no adapter registered for #{url.inspect}")
+    end
+  end
+
+  # The URL looked supported, but the remote document did not contain the
+  # slide data we expected. Usually means the site changed its markup.
+  class ExtractionError < Error; end
+
+  # Something went wrong while talking to the remote host.
+  class FetchError < Error
+    attr_reader :url, :status
+
+    def initialize(message, url: nil, status: nil)
+      @url = url
+      @status = status
+      super(message)
+    end
+  end
+
+  # The remote host answered, but with a non-success status.
+  class ResponseError < FetchError; end
+
+  # We followed more redirects than the fetcher allows.
+  class TooManyRedirectsError < FetchError; end
+
+  # The host's robots.txt does not allow this client to fetch this path — or
+  # could not be read, which comes to the same thing.
+  #
+  # A FetchError because it is a request that did not happen, which also means
+  # an optional extra like an oEmbed lookup degrades instead of failing the
+  # whole fetch.
+  class RobotsDisallowedError < FetchError; end
+end

data/lib/gento/fetcher.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "resolv"
+require "ipaddr"
+
+module Gento
+  # The HTTP seam.
+  #
+  # Everything the library does over the network goes through one of these, so
+  # a host that has its own HTTP stack (a Worker's `fetch`, a Faraday
+  # connection, a test double) can supply an adapter instead of dragging
+  # net/http along.
+  #
+  # Implementations must return a Gento::Response and raise
+  # Gento::FetchError on transport failure.
+  class Fetcher
+    def get(url, headers: {})
+      raise NotImplementedError, "#{self.class} must implement #get"
+    end
+  end
+
+  # Default fetcher, built on net/http from the standard library.
+  class NetHttpFetcher < Fetcher
+    DEFAULT_USER_AGENT =
+      "gento/#{VERSION} (+https://github.com/ngram/gento)".freeze
+
+    # Ranges that a public slide host has no business resolving to. Checked
+    # before connecting so a redirect cannot walk us into the private network.
+    BLOCKED_RANGES = [
+      IPAddr.new("0.0.0.0/8"), IPAddr.new("10.0.0.0/8"),
+      IPAddr.new("127.0.0.0/8"), IPAddr.new("169.254.0.0/16"),
+      IPAddr.new("172.16.0.0/12"), IPAddr.new("192.168.0.0/16"),
+      IPAddr.new("100.64.0.0/10"), IPAddr.new("::1/128"),
+      IPAddr.new("fc00::/7"), IPAddr.new("fe80::/10")
+    ].freeze
+
+    attr_reader :user_agent, :open_timeout, :read_timeout, :max_redirects, :proxy
+
+    def initialize(user_agent: DEFAULT_USER_AGENT, open_timeout: 5, read_timeout: 10,
+                   max_redirects: 5, block_private_addresses: true, proxy: :env)
+      @user_agent = user_agent
+      @open_timeout = open_timeout
+      @read_timeout = read_timeout
+      @max_redirects = max_redirects
+      @block_private_addresses = block_private_addresses
+      @proxy = proxy == :env ? proxy_from_env : normalize_proxy(proxy)
+      super()
+    end
+
+    def get(url, headers: {})
+      uri = normalize_uri(url)
+      redirects = 0
+
+      loop do
+        response = request(uri, headers)
+        location = response["location"]
+
+        return build_response(response, uri) unless redirect?(response) && location
+
+        redirects += 1
+        if redirects > max_redirects
+          raise TooManyRedirectsError.new("more than #{max_redirects} redirects", url: url)
+        end
+
+        uri = normalize_uri(URI.join(uri, location))
+      end
+    end
+
+    private
+
+    def request(uri, headers)
+      addresses = resolve(uri.hostname)
+      guard_address!(uri, addresses)
+
+      connection(uri, addresses).start do |http|
+        http.request(Net::HTTP::Get.new(uri, request_headers(headers)))
+      end
+    rescue Timeout::Error, SystemCallError, IOError, OpenSSL::SSL::SSLError,
+           Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError => e
+      raise FetchError.new("could not fetch #{uri}: #{e.class}: #{e.message}", url: uri.to_s)
+    end
+
+    # Built explicitly rather than through Net::HTTP.start's keyword form:
+    # there the proxy arguments are positional, so passing them as keywords
+    # puts them in the options hash where they are silently ignored, leaving
+    # every request unproxied. Passing nil for the address disables proxying.
+    def connection(uri, addresses)
+      http = Net::HTTP.new(uri.hostname, uri.port, *proxy_parts(addresses))
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = open_timeout
+      http.read_timeout = read_timeout
+      http
+    end
+
+    # net/http only consults the lowercase `http_proxy`, and never for HTTPS,
+    # so environments that set HTTPS_PROXY — most corporate networks and every
+    # sandboxed CI runner — go unproxied and fail. Read both spellings.
+    def proxy_from_env
+      value = ENV["HTTPS_PROXY"] || ENV["https_proxy"] || ENV["HTTP_PROXY"] || ENV.fetch("http_proxy", nil)
+      normalize_proxy(value)
+    end
+
+    def normalize_proxy(value)
+      return nil if value.nil? || value.to_s.empty?
+
+      uri = value.is_a?(URI) ? value : URI.parse(value.to_s)
+      uri.host ? uri : nil
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    # Identity encoding: we decode charsets ourselves and have no reason to
+    # also own gzip handling.
+    def request_headers(headers)
+      { "user-agent" => user_agent, "accept-encoding" => "identity" }
+        .merge(headers.transform_keys { |key| key.to_s.downcase })
+    end
+
+    def build_response(response, uri)
+      status = response.code.to_i
+      unless status.between?(200, 299)
+        raise ResponseError.new("#{uri} responded #{status}", url: uri.to_s, status: status)
+      end
+
+      headers = response.each_header.to_h
+      Response.new(status: status, body: response.body.to_s, url: uri.to_s, headers: headers)
+    end
+
+    def redirect?(response)
+      response.is_a?(Net::HTTPRedirection)
+    end
+
+    def normalize_uri(url)
+      uri = url.is_a?(URI) ? url : URI.parse(url.to_s)
+      unless uri.is_a?(URI::HTTP)
+        raise FetchError.new("only http(s) URLs are supported, got #{url.inspect}", url: url.to_s)
+      end
+
+      uri
+    end
+
+    # Best-effort SSRF guard. There is an unavoidable gap between resolving a
+    # name here and net/http resolving it again when it connects; closing it
+    # properly needs socket-level pinning. This still stops the ordinary case
+    # of a redirect pointed at an internal address.
+    def guard_address!(uri, addresses)
+      return unless @block_private_addresses
+
+      blocked = blocked_address(addresses)
+      return unless blocked
+
+      raise FetchError.new("refusing to connect to private address #{blocked} (#{uri.hostname})",
+                           url: uri.to_s)
+    end
+
+    # A proxy exists to reach the outside world, and loopback or private
+    # addresses are not out there. Routing them through one is how a request
+    # to a service on this machine ends up answered by the proxy instead.
+    def proxy_parts(addresses)
+      return [nil, nil, nil, nil] if proxy.nil? || blocked_address(addresses)
+
+      [proxy.hostname, proxy.port, proxy.user, proxy.password]
+    end
+
+    def blocked_address(addresses)
+      addresses.find { |address| BLOCKED_RANGES.any? { |range| range.include?(address) } }
+    end
+
+    def resolve(host)
+      return [IPAddr.new(host)] if ip_literal?(host)
+
+      Resolv.getaddresses(host).filter_map do |address|
+        IPAddr.new(address)
+      rescue IPAddr::Error
+        nil
+      end
+    end
+
+    def ip_literal?(host)
+      IPAddr.new(host)
+      true
+    rescue IPAddr::Error
+      false
+    end
+  end
+end