ruby-laya 0.1.0

data/lib/laya/configuration.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "monitor"
+
+module Laya
+  # How this process loads and runs checkpoints.
+  #
+  #   Laya.configure do |config|
+  #     config.device = "coreml"
+  #     config.preload = true
+  #   end
+  #
+  # Settings apply to the shared client that {Laya.ask} and {Laya::Decision} use. Change them
+  # before the first decision; afterwards, call {Laya.reset!} to rebuild the client.
+  class Configuration
+    # "cpu", "coreml", "cuda", or nil to let the runtime decide.
+    attr_accessor :device
+    # ONNX Runtime providers, overriding `device` when set.
+    attr_accessor :providers
+    # Intra-op threads. Nil lets ONNX Runtime choose.
+    attr_accessor :threads
+    # Hugging Face token, for a private or gated export.
+    attr_accessor :token
+    # Pin every decision to one checkpoint instead of routing per request.
+    attr_accessor :model
+    # How many checkpoints stay resident.
+    attr_accessor :max_loaded
+    # Build every checkpoint at startup rather than on first use.
+    attr_accessor :preload
+    # A language code, or a callable taking the state, used ahead of the built-in detection.
+    attr_accessor :lang_guess
+
+    def initialize
+      @max_loaded = Router::DEFAULT_MAX_LOADED
+      @preload = false
+    end
+
+    # The options a Router is built from.
+    def router_options
+      { device: device, providers: providers, threads: threads, token: token,
+        max_loaded: max_loaded, preload: preload, lang_guess: lang_guess }.compact
+    end
+  end
+
+  class << self
+    # The shared configuration.
+    def config
+      @config ||= Configuration.new
+    end
+
+    def configure
+      yield config
+      reset!
+      config
+    end
+
+    # The router every decision goes through unless it was given its own.
+    def client
+      @client_lock ||= Monitor.new
+      @client_lock.synchronize { @client ||= Router.new(**config.router_options) }
+    end
+
+    # Drop the shared client, closing whatever it had resident. The next decision rebuilds it.
+    def reset!
+      @client_lock ||= Monitor.new
+      @client_lock.synchronize do
+        @client&.close
+        @client = nil
+      end
+    end
+  end
+end

data/lib/laya/decision.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require_relative "questions"
+
+module Laya
+  # A named set of typed questions, declared once and asked many times.
+  #
+  #   class TicketTriage < Laya::Decision
+  #     choice :department, "Which team should handle this?",
+  #            billing: "invoices, payments, refunds",
+  #            technical: "bugs, outages, system errors",
+  #            other: "everything else"
+  #
+  #     score :urgency, "How urgent is this?",
+  #           levels: ["not urgent", "soon", "critical deadline"]
+  #
+  #     noul :churn_risk, "Does the customer threaten to cancel?"
+  #   end
+  #
+  #   triage = TicketTriage.decide(email)
+  #   triage.department             # => #<Laya::Answer::Choice billing 95.9%>
+  #   triage.department == :billing # => true
+  #   triage.urgency.score          # => 1.36
+  #   triage.churn_risk?            # => true
+  #
+  # Every question becomes a reader named after it, and every noul also gets a predicate. The
+  # underlying {Result}, with its routing and token usage, is on `#result`.
+  class Decision
+    class << self
+      # The questions this decision asks, in declaration order, in the shape the runtime takes.
+      def questions
+        @questions ||= superclass.respond_to?(:questions) ? superclass.questions.dup : {}
+      end
+
+      # Pin this decision to one checkpoint instead of routing per request.
+      #
+      #   model "multilingual"
+      def model(name = nil)
+        @model = name unless name.nil?
+        @model || (superclass.respond_to?(:model) ? superclass.model : nil)
+      end
+
+      # Register a question already in the runtime's shape. This is what {define} uses, and the
+      # way to declare a question whose text is built at load time rather than written out.
+      # The key is kept as given, so a question set with string ids answers with string ids and
+      # matches what the raw path would return.
+      def question(name, definition)
+        declare(name, definition)
+        define_method("#{name}?") { self[name].true? } if Util.get(definition, "type").to_s == "noul"
+        name
+      end
+
+      # A decision class from a question hash, for question sets that are generated or shipped.
+      #
+      #   Triage = Laya::Decision.define(Laya.triage_questions)
+      def define(questions, model: nil)
+        Class.new(self) do
+          self.model(model) if model
+          questions.each { |name, definition| question(name, definition) }
+        end
+      end
+
+      def choice(name, instructions, criteria = nil, **labels)
+        declare(name, Questions.choice(instructions, criteria, **labels))
+      end
+
+      def score(name, instructions, levels:)
+        declare(name, Questions.score(instructions, levels))
+      end
+
+      def noul(name, instructions, yes: nil, no: nil)
+        declare(name, Questions.noul(instructions, yes: yes, no: no))
+        define_method("#{name}?") { self[name].true? }
+      end
+
+      # Ask every question about `state`.
+      #
+      # `client` is anything answering `predict(state, questions, **options)`: an {Agent}, a
+      # {Router}, or your own double in a test. It defaults to the shared client.
+      def decide(state, client: nil, **options)
+        options[:model] ||= model if model && (client || Laya.client).is_a?(Router)
+        result = (client || Laya.client).predict(state, questions, **options)
+        new(result)
+      end
+
+      private
+
+      def declare(name, question)
+        questions[name] = question
+        define_method(name.to_s) { self[name] }
+        name
+      end
+
+      def inherited(subclass)
+        super
+        subclass.instance_variable_set(:@questions, questions.dup)
+      end
+    end
+
+    attr_reader :result
+
+    def initialize(result)
+      @result = result
+    end
+
+    # The answer to one question, by the name it was declared with.
+    def [](name)
+      result[name]
+    end
+
+    def to_h = result.to_h
+    def usage = result.usage
+    def routing = result.routing
+
+    def inspect
+      answered = self.class.questions.keys.map { |name| "#{name}=#{self[name]}" }
+      "#<#{self.class.name || 'Decision'} #{answered.join(' ')}>"
+    end
+  end
+end

data/lib/laya/email.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+module Laya
+  # Cleaning and structuring email before a checkpoint reads it.
+  #
+  # The markers cover English, Portuguese and Spanish mail clients. Quoted history is often a
+  # different request than the new message, and it weighs on the answer just as heavily, so it
+  # has to go; a footer that survives is harmless by comparison, which is why the cleaning errs
+  # towards keeping text.
+  module Email
+    QUOTE_HEADERS = [
+      /\A\s*On .{0,300}wrote:\s*\z/i,
+      # "Em resposta ao que você escreveu:" is body text; a client's attribution carries a date.
+      /\A\s*Em (?=.*\d).{0,300}escreveu:\s*\z/i,
+      /\A\s*El (?=.*\d).{0,300}escribi[óo]:\s*\z/i,
+      /\A\s*-{2,}\s*(Original|Forwarded) Message\s*-{2,}/i,
+      /\A\s*-{2,}\s*(Mensagem (original|encaminhada)|Mensaje (original|reenviado))\s*-{2,}/i,
+      /\A\s*_{8,}\s*\z/,
+      /\A\s*From:\s.+\z/i,
+      # `De:` also opens ordinary Portuguese and Spanish lines ("De: 10/09 a 15/09"), so this one
+      # is only a header when it carries an address.
+      /\A\s*De:\s.*[@<]/i
+    ].freeze
+
+    # Gmail wraps a long attribution, leaving `someone@x.com> escreveu:` alone on the next line.
+    # That tail cuts too, and takes the `On/Em/El ...` head it belongs to with it.
+    ATTRIBUTION_TAIL = /\A.{0,120}\S@\S+\s+(wrote|escreveu|escribi[óo]):\s*\z/i
+    ATTRIBUTION_HEAD = /\A\s*(On|Em|El) (?=.*\d)/i
+
+    # Exchange often leaves the address out of Outlook's reply header ("De: Maria Souza"), so a
+    # bare `De:` only cuts when the header's own `Enviado:` or a dated `Data:` line follows it.
+    HEADER_FROM_NAME = /\A\s*De:\s+\S/i
+    HEADER_NEXT = /\A\s*(Enviad[oa]( em| el)?:\s|(Data|Fecha):\s.*\d{4})/i
+
+    SIGNATURE_MARKERS = [
+      /\A\s*--\s*\z/,
+      /\A\s*(best|kind|warm|many thanks|thanks|thank you|regards|cheers|sincerely)[\w ,!.]*\z/i,
+      /\A\s*sent from my (iphone|android|mobile|ipad)/i,
+      # Portuguese and Spanish sign-offs match only on their own: "Obrigado pelo retorno, mas ..."
+      # is a request, so unlike the English marker no trailing words are allowed.
+      /\A\s*(atenciosamente|att|abraços?|abs|um abraço|cordialmente|grat[oa]|(muito )?obrigad[oa]s?
+        ( desde já| pela atenção)?|(com os melhores )?cumprimentos|saudações|
+        (un )?saludos?( cordiales)?|atentamente|(muchas )?gracias( de antemano)?)[\s,!.]*\z/xi
+    ].freeze
+
+    # Mobile and mail-app footers. Only a line that is nothing but the footer matches, and such a
+    # line may run to 60 characters: Samsung's default is longer than any sign-off.
+    DEVICE = "iphone|ipad|android|ios|celular|telemóvel|móvil|galaxy|smartphone|samsung|tablet|" \
+             "outlook|yahoo|mail|e-?mail|gmail|windows"
+    DEVICE_FOOTER = Regexp.new(
+      "\\A\\s*((enviad[oa] (do|pelo|pela|via|desde|a partir do)( meu| minha| mi)?|sent from( my)?) " \
+      "(#{DEVICE})( (#{DEVICE}|para|for|no|na|\\d+))*|(obter o|get) outlook (para|for) (ios|android))[\\s.!]*\\z",
+      Regexp::IGNORECASE
+    )
+
+    # Confidentiality footers. The Portuguese and Spanish patterns are tied to "this message"
+    # rather than to the bare word `confidencial`, which a sender's own request uses just as
+    # often ("preciso do contrato confidencial").
+    DISCLAIMER_PARTS = [
+      "confidential",
+      "intended (solely )?for the (use of the )?(named )?(addressee|recipient)",
+      "if you (have )?received this (e-?mail|message) in error",
+      "\\b(esta|este) (mensagem|e-?mail|mensaje|correo)\\b[^.]{0,80}(confidencia|sigilos|privilegiad)",
+      "\\b(uso exclusivo|exclusivamente|únicamente|unicamente)\\b[^.]{0,30}" \
+      "(destinatári|destinatari|pessoa|persona|entidade|entidad)",
+      "\\b(recebeu|recebido|receber) (esta|este) (mensagem|e-?mail)\\b[^.]{0,20} por (engano|erro)",
+      "\\b(ha recibido|recibió|recibe) (este|esta) (mensaje|correo)\\b[^.]{0,20} por error",
+      # the "think before printing" footer, tied to its environmental ending rather than to
+      # `antes de imprimir`, which a request uses too ("antes de imprimir o boleto, confira")
+      "\\bantes de imprimir\\b[^.]{0,100}(meio ambiente|medio ambiente|natureza|planeta|realmente necess)",
+      "\\b(meio|medio) ambiente\\b[^.]{0,30}antes de imprimir"
+    ].freeze
+    DISCLAIMER = Regexp.new("(#{DISCLAIMER_PARTS.join('|')})", Regexp::IGNORECASE)
+
+    SENTENCE = /(?<=[.!?])\s+/
+
+    # A signature marker only counts on a short line; a device footer gets more room.
+    SIGNATURE_MAX = 40
+    DEVICE_FOOTER_MAX = 60
+
+    module_function
+
+    # Remove quoted history, signatures and disclaimers, keeping the sender's own request.
+    def clean_email_body(body, max_chars: 3000)
+      lines = strip_quoted_history(normalise(body).split("\n", -1))
+      lines = lines.first(signature_cut(lines))
+      paragraphs = lines.join("\n").split(/\n\s*\n/).map { |paragraph| strip_disclaimer(paragraph) }
+      paragraphs.map(&:strip).reject(&:empty?).join("\n\n").gsub(/[ \t]+/, " ")[0, max_chars].to_s
+    end
+
+    def normalise(body)
+      (body || "").gsub("\r\n", "\n").gsub("\r", "\n").gsub("\\n", "\n")
+    end
+
+    # Everything up to the first quote header, minus the quoted lines themselves.
+    def strip_quoted_history(source)
+      kept = []
+      source.each_with_index do |line, i|
+        break if !kept.empty? && QUOTE_HEADERS.any? { |pattern| line.match?(pattern) }
+        break if !kept.empty? && outlook_header?(line, source[i + 1])
+
+        if ATTRIBUTION_TAIL.match?(line) && !kept.empty?
+          kept.pop if ATTRIBUTION_HEAD.match?(kept.last)
+          break
+        end
+        next if line.lstrip.start_with?(">")
+
+        kept << line.rstrip
+      end
+      kept
+    end
+
+    def outlook_header?(line, following)
+      HEADER_FROM_NAME.match?(line) && following && HEADER_NEXT.match?(following)
+    end
+
+    # Where the signature starts, or the end of the message when there is none. Only the last
+    # part of a message is considered, so a "Thanks" opening line is never mistaken for a sign-off.
+    def signature_cut(lines)
+      first = (lines.length * 0.6).to_i.clamp(1, [lines.length - 8, 1].max)
+      (first...lines.length).each do |i|
+        length = lines[i].strip.length
+        signature = length <= SIGNATURE_MAX && SIGNATURE_MARKERS.any? { |pattern| lines[i].match?(pattern) }
+        return i if signature || (length <= DEVICE_FOOTER_MAX && DEVICE_FOOTER.match?(lines[i]))
+      end
+      lines.length
+    end
+
+    # Drop boilerplate from one paragraph, sentence by sentence.
+    #
+    # The whole paragraph goes only when every sentence in it is boilerplate. A footer that runs
+    # on without a blank line used to take the sender's actual request with it, which is worse
+    # than leaving a boilerplate line behind.
+    def strip_disclaimer(paragraph)
+      return paragraph unless paragraph.match?(DISCLAIMER) # keep the original line structure
+
+      sentences = paragraph.split(SENTENCE).map(&:strip).reject(&:empty?)
+      pieces = sentences.flat_map { |s| s.match?(DISCLAIMER) ? split_fused_lines(s) : [s] }
+      pieces.grep_v(DISCLAIMER).join(" ")
+    end
+
+    # Split a boilerplate sentence where a new sentence starts on a new line.
+    #
+    # An unpunctuated request glued to a disclaimer ("locked\nThis email is...") splits, because
+    # the next line starts with a capital; a wrapped continuation ("are\nconfidential") does not,
+    # so a wrapped footer still drops whole.
+    def split_fused_lines(sentence)
+      return [sentence] unless sentence.include?("\n")
+
+      pieces = []
+      current = +""
+      sentence.split("\n").each do |line|
+        if current.empty? || !starts_new_sentence?(line)
+          current << "\n" unless current.empty?
+          current << line
+        else
+          pieces << current
+          current = +line
+        end
+      end
+      pieces << current unless current.empty?
+      pieces.map(&:strip).reject(&:empty?)
+    end
+
+    # True when the first letter is a capital: a fresh sentence rather than a wrapped line. Lines
+    # in uncased scripts never start a new piece, so wrapped boilerplate there drops whole.
+    def starts_new_sentence?(line)
+      letter = line.each_char.find { |char| char.match?(/\p{L}/) }
+      return false unless letter
+
+      letter == letter.upcase && letter != letter.downcase
+    end
+
+    # A state ready for the email presets: subject, cleaned body, and whatever else you pass.
+    def email_state(subject, body, sender: nil, clean: true, **extra)
+      state = { "subject" => (subject || "").strip,
+                "body" => clean ? clean_email_body(body) : (body || "") }
+      state["from"] = sender if sender && !sender.to_s.empty?
+      extra.each { |key, value| state[key.to_s] = value unless value.nil? }
+      state
+    end
+
+    def email_questions(categories = nil)
+      Presets.email_questions(categories)
+    end
+  end
+end

data/lib/laya/errors.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Laya
+  # Base class for every error raised by Laya.
+  class Error < StandardError; end
+
+  # A model path, subfolder or required checkpoint file could not be found.
+  class ModelNotFoundError < Error; end
+
+  # The checkpoint on disk is not a Laya decision model (missing config, weights or a
+  # shape that does not match the architecture).
+  class IncompatibleModelError < Error; end
+
+  # A network download from the Hugging Face Hub failed.
+  class DownloadError < Error; end
+end

data/lib/laya/hub.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+require "fileutils"
+
+module Laya
+  # Downloads checkpoint files from the Hugging Face Hub into the standard Hugging Face cache, so
+  # `HF_HOME`, `HF_HUB_CACHE`, `HF_HUB_OFFLINE` and `HF_TOKEN` all behave as they do for any other
+  # Hub client, and a cached snapshot keeps working without a network.
+  #
+  # Files land in `<cache>/models--<org>--<name>/snapshots/<revision sha>/<path>`, the layout the
+  # Hub's own tooling reads.
+  module Hub
+    MAX_REDIRECTS = 5
+    DEFAULT_REVISION = "main"
+
+    # Reports download progress. Replace it to drive your own progress bar, or set it to nil.
+    #
+    #   Laya::Hub.progress = ->(path, done, total) { ... }
+    class << self
+      attr_writer :progress
+
+      def progress
+        return @progress if defined?(@progress)
+
+        @progress = method(:report_progress)
+      end
+    end
+
+    module_function
+
+    def endpoint
+      ENV.fetch("HF_ENDPOINT", "https://huggingface.co").sub(%r{/+\z}, "")
+    end
+
+    def offline?
+      %w[1 true yes on].include?(ENV.fetch("HF_HUB_OFFLINE", "").downcase)
+    end
+
+    # An empty variable counts as unset, so exporting HF_TOKEN= does not hide the other name.
+    def token
+      [ENV.fetch("HF_TOKEN", nil), ENV.fetch("HUGGING_FACE_HUB_TOKEN", nil)].find { |value| present?(value) }
+    end
+
+    def cache_dir
+      return File.expand_path(ENV["HF_HUB_CACHE"]) if present?(ENV["HF_HUB_CACHE"])
+      return File.join(File.expand_path(ENV["HF_HOME"]), "hub") if present?(ENV["HF_HOME"])
+
+      File.join(Dir.home, ".cache", "huggingface", "hub")
+    end
+
+    def present?(value)
+      value && !value.empty?
+    end
+
+    def repo_dir(repo_id, cache_dir: nil)
+      File.join(cache_dir || Hub.cache_dir, "models--#{repo_id.gsub('/', '--')}")
+    end
+
+    # Download every file under `subfolder` matching `allow_patterns` and return the local
+    # directory holding them.
+    #
+    # Files already present are not fetched again. When the Hub cannot be reached, the newest
+    # cached snapshot is used instead, so an offline process keeps working.
+    def snapshot(repo_id, subfolder: nil, allow_patterns: nil, revision: DEFAULT_REVISION,
+                 token: nil, cache_dir: nil)
+      token ||= Hub.token
+      root = repo_dir(repo_id, cache_dir: cache_dir)
+      sha = resolve_revision(repo_id, revision, token: token, root: root)
+      snapshot_dir = File.join(root, "snapshots", sha)
+      target = subfolder ? File.join(snapshot_dir, subfolder) : snapshot_dir
+
+      if sha == cached_revision(root, revision) && offline?
+        raise DownloadError, "HF_HUB_OFFLINE is set and #{repo_id} is not cached" unless File.directory?(target)
+
+        return target
+      end
+
+      prefix = subfolder ? "#{subfolder}/" : ""
+      wanted = Array(allow_patterns).map { |pattern| prefix + pattern }
+      files = filter(list_files(repo_id, revision: sha, token: token), wanted)
+      raise DownloadError, "no files in #{repo_id.inspect} match #{wanted.inspect}" if files.empty?
+
+      files.each do |path|
+        local = File.join(snapshot_dir, path)
+        next if File.file?(local) && File.size(local).positive?
+
+        download(repo_id, path, local, revision: sha, token: token)
+      end
+      write_ref(root, revision, sha)
+      target
+    end
+
+    # The commit the revision points at, or the cached one when the Hub is unreachable.
+    def resolve_revision(repo_id, revision, token: nil, root: nil)
+      return cached_revision!(root, revision, repo_id) if offline?
+
+      uri = URI("#{endpoint}/api/models/#{repo_id}/revision/#{URI.encode_www_form_component(revision)}")
+      sha = JSON.parse(get(uri, token: token)).fetch("sha")
+      raise DownloadError, "#{repo_id} revision #{revision.inspect} has no commit sha" unless sha
+
+      sha
+    rescue JSON::ParserError, KeyError => e
+      raise DownloadError, "unexpected response resolving #{repo_id.inspect}: #{e.message}"
+    rescue DownloadError => e
+      cached = cached_revision(root, revision)
+      raise e unless cached
+
+      warn "[laya] #{e.message}; using the cached snapshot #{cached[0, 7]}"
+      cached
+    end
+
+    def cached_revision(root, revision)
+      ref = File.join(root.to_s, "refs", revision.to_s)
+      File.file?(ref) ? File.read(ref).strip : nil
+    end
+
+    def cached_revision!(root, revision, repo_id)
+      cached_revision(root, revision) ||
+        raise(DownloadError, "HF_HUB_OFFLINE is set and #{repo_id} is not cached in #{root}")
+    end
+
+    def write_ref(root, revision, sha)
+      ref = File.join(root, "refs", revision.to_s)
+      FileUtils.mkdir_p(File.dirname(ref))
+      File.write(ref, sha)
+    end
+
+    def list_files(repo_id, revision: DEFAULT_REVISION, token: nil)
+      uri = URI("#{endpoint}/api/models/#{repo_id}/revision/#{URI.encode_www_form_component(revision)}")
+      JSON.parse(get(uri, token: token)).fetch("siblings", []).map { |sibling| sibling["rfilename"] }
+    rescue JSON::ParserError => e
+      raise DownloadError, "unexpected response listing #{repo_id.inspect}: #{e.message}"
+    end
+
+    # Hub glob semantics: `*` matches across path separators too.
+    def filter(files, patterns)
+      return files if patterns.nil? || patterns.empty?
+
+      files.select { |file| patterns.any? { |pattern| File.fnmatch(pattern, file, File::FNM_DOTMATCH) } }
+    end
+
+    def download(repo_id, path, local, revision: DEFAULT_REVISION, token: nil)
+      uri = URI("#{endpoint}/#{repo_id}/resolve/#{URI.encode_www_form_component(revision)}/#{path}")
+      FileUtils.mkdir_p(File.dirname(local))
+      partial = "#{local}.incomplete"
+      done = 0
+      File.open(partial, "wb") do |file|
+        get(uri, token: token) do |chunk, total|
+          file.write(chunk)
+          done += chunk.bytesize
+          Hub.progress&.call(path, done, total)
+        end
+      end
+      File.rename(partial, local)
+      local
+    rescue StandardError => e
+      FileUtils.rm_f(partial.to_s)
+      raise e
+    end
+
+    def report_progress(path, done, total)
+      return unless $stderr.tty?
+
+      percent = total.to_i.positive? ? format(" %3d%%", 100 * done / total) : ""
+      $stderr.print(format("\r[laya] %s%s %.0f MB", File.basename(path), percent, done / 1e6))
+      $stderr.print("\n") if total.to_i.positive? && done >= total
+    end
+
+    # GET with redirects, streaming to the block when one is given.
+    def get(uri, token: nil, redirects: 0, &block)
+      raise DownloadError, "too many redirects for #{uri}" if redirects > MAX_REDIRECTS
+
+      request = Net::HTTP::Get.new(uri)
+      request["User-Agent"] = "ruby-laya/#{Laya::VERSION}"
+      request["Authorization"] = "Bearer #{token}" if token && uri.host == URI(endpoint).host
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+                                          open_timeout: 30, read_timeout: 300) do |http|
+        http.request(request) do |response|
+          case response
+          when Net::HTTPRedirection
+            return get(URI.join(uri, response["location"]), token: token, redirects: redirects + 1, &block)
+          when Net::HTTPSuccess
+            return response.body unless block
+
+            total = response["content-length"].to_i
+            response.read_body { |chunk| block.call(chunk, total) }
+            return nil
+          when Net::HTTPUnauthorized, Net::HTTPForbidden
+            raise DownloadError, "access denied for #{uri} (HTTP #{response.code}); " \
+                                 "set HF_TOKEN for gated or private repositories"
+          when Net::HTTPNotFound
+            raise DownloadError, "not found: #{uri} (HTTP 404)"
+          else
+            raise DownloadError, "HTTP #{response.code} fetching #{uri}"
+          end
+        end
+      end
+    rescue SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError, IOError => e
+      raise DownloadError, "could not fetch #{uri}: #{e.class}: #{e.message}"
+    end
+  end
+end