relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/nist/scraper.rb

@@ -0,0 +1,293 @@
+require "yaml"
+require "net/http"
+
+module Relaton
+  module Nist
+    class Scraper
+      extend Core::DateParser
+
+      class << self
+        DOMAIN = "https://csrc.nist.gov".freeze
+
+        # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+
+        # Parse page.
+        # @param hit_data [Hash]
+        # @return [Relaton::Nist::ItemData]
+        def parse_page(hit_data)
+          hit_data[:path] ? fetch_gh(hit_data) : parse_json(hit_data)
+        end
+
+        def fetch_gh(hit_data)
+          uri = URI.parse "#{HitCollection::GHNISTDATA}#{hit_data[:path]}"
+          yaml = Net::HTTP.get(uri)
+          Item.from_yaml(yaml)
+        end
+
+        def parse_json(hit_data)
+          item_data = from_json hit_data
+          titles = fetch_titles(hit_data)
+          item_data[:fetched] = ::Date.today.to_s
+          item_data[:type] = "standard"
+          item_data[:title] = titles
+          ItemData.new(**item_data)
+        end
+
+        private
+
+        def from_json(hit_data)
+          json = hit_data[:json]
+          {
+            source: fetch_link(json),
+            docidentifier: fetch_docid(hit_data),
+            date: fetch_dates(json, hit_data[:release_date]),
+            contributor: fetch_contributors(json),
+            edition: fetch_edition(json),
+            language: [json["language"]],
+            script: [json["script"]],
+            status: fetch_status(json),
+            copyright: fetch_copyright(json["published-date"]),
+            relation: fetch_relations_json(hit_data),
+            place: fetch_place,
+            keyword: fetch_keywords(json),
+            ext: Ext.new(
+              doctype: Doctype.new(content: "standard"),
+              flavor: "nist",
+              commentperiod: fetch_commentperiod_json(json),
+            ),
+          }
+        end
+
+        # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+        # Fetch docid.
+        # @param hit [Hash]
+        # @return [Array<Bib::Docidentifier>]
+        def fetch_docid(hit)
+          # All ids share the Nist::Docidentifier collection type so YAML
+          # serialization stays valid; the DOI is not canonical NIST pubid form,
+          # so the class keeps it as the raw string (see Docidentifier#content=).
+          ids = [Docidentifier.new(content: hit[:code], type: "NIST", primary: true)]
+          doi = hit[:json]["doi"]&.split("/")&.last
+          ids << Docidentifier.new(content: doi, type: "DOI") if doi
+          ids
+        end
+
+        # Fetch status.
+        # @param doc [Hash]
+        # @return [Bib::Status]
+        def fetch_status(doc)
+          stage = doc["status"]
+          subst = doc["substage"]
+          iter = iteration(doc["iteration"])
+          Bib::Status.new(
+            stage: Bib::Status::Stage.new(content: stage),
+            substage: subst ? Bib::Status::Stage.new(content: subst) : nil,
+            iteration: iter.to_s,
+          )
+        end
+
+        def iteration(iter)
+          case iter
+          when "initial", "ipd" then 1
+          when /(\d)pd/i then $1
+          else iter
+          end
+        end
+
+        # Fetch titles.
+        # @param hit_data [Hash]
+        # @return [Array<Bib::Title>]
+        def fetch_titles(hit_data)
+          [Bib::Title.new(content: hit_data[:title], language: "en", script: "Latn")]
+        end
+
+        # Fetch dates
+        # @param doc [Hash]
+        # @param release_date [Date]
+        # @return [Array<Bib::Date>]
+        def fetch_dates(doc, release_date) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+          dates = [Bib::Date.new(type: "published", at: release_date.to_s)]
+
+          issued = safe_parse_date doc["issued-date"]
+          updated = safe_parse_date doc["updated-date"]
+          dates << Bib::Date.new(type: "updated", at: updated.to_s) if updated
+          obsoleted = safe_parse_date doc["obsoleted-date"]
+          dates << Bib::Date.new(type: "obsoleted", at: obsoleted.to_s) if obsoleted
+          dates << Bib::Date.new(type: "issued", at: issued.to_s) if issued
+          dates
+        end
+
+        def safe_parse_date(str)
+          return unless str
+
+          parse_date(str, str: false)
+        end
+
+        # @param doc [Hash]
+        # @return [Array<Bib::Contributor>]
+        def fetch_contributors(doc)
+          contribs = []
+          contribs += contributors_json(
+            doc["authors"], "author", doc["language"], doc["script"],
+          )
+          contribs + contributors_json(
+            doc["editors"], "editor", doc["language"], doc["script"],
+          )
+        end
+
+        # @param doc [Array<Hash>]
+        # @param role [String]
+        # @return [Array<Bib::Contributor>]
+        def contributors_json(doc, role, lang = "en", script = "Latn") # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+          doc.map do |contr|
+            if contr["affiliation"]
+              abbrev = if contr["affiliation"]["acronym"]
+                         Bib::LocalizedString.new(content: contr["affiliation"]["acronym"])
+                       end
+              org = Bib::Organization.new(
+                name: [Bib::TypedLocalizedString.new(content: contr["affiliation"]["name"])],
+                abbreviation: abbrev,
+              )
+            end
+            if contr["surname"]
+              affiliation = []
+              affiliation << Bib::Affiliation.new(organization: org) if org
+              entity = Bib::Person.new(
+                name: full_name(contr, lang, script), affiliation: affiliation,
+              )
+              Bib::Contributor.new(
+                person: entity,
+                role: [Bib::Contributor::Role.new(type: role)],
+              )
+            elsif org
+              Bib::Contributor.new(
+                organization: org,
+                role: [Bib::Contributor::Role.new(type: role)],
+              )
+            end
+          end.compact
+        end
+
+        # @param name [Hash]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Bib::FullName]
+        def full_name(name, lang, script)
+          Bib::FullName.new(
+            surname: Bib::LocalizedString.new(content: name["surname"], language: lang, script: script),
+            forename: name_parts_forename(name["givenName"], lang, script),
+            addition: name_parts(name["suffix"], lang, script),
+            prefix: name_parts(name["title"], lang, script),
+            completename: Bib::LocalizedString.new(content: name["fullName"], language: lang, script: script),
+          )
+        end
+
+        # @param part [String, NilClass]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Array<Bib::LocalizedString>]
+        def name_parts(part, lang, script)
+          return [] unless part
+
+          [Bib::LocalizedString.new(content: part, language: lang, script: script)]
+        end
+
+        # @param part [String, NilClass]
+        # @param lang [String]
+        # @param script [String]
+        # @return [Array<Bib::FullNameType::Forename>]
+        def name_parts_forename(part, lang, script)
+          return [] unless part
+
+          [Bib::FullNameType::Forename.new(content: part, language: lang, script: script)]
+        end
+
+        # @param doc [Hash]
+        # @return [String, NilClass]
+        def fetch_edition(doc)
+          return unless doc["edition"] || doc["revision"]
+
+          rev = doc["edition"] || doc["revision"]
+          Bib::Edition.new(content: "Revision #{rev}")
+        end
+
+        # Fetch copyright.
+        # @param doc [String]
+        # @return [Array<Bib::Copyright>]
+        def fetch_copyright(doc)
+          from = doc&.match(/\d{4}/)&.to_s
+          owner = Bib::ContributionInfo.new(
+            organization: Bib::Organization.new(
+              name: [Bib::TypedLocalizedString.new(content: "National Institute of Standards and Technology")],
+              abbreviation: Bib::LocalizedString.new(content: "NIST"),
+              uri: [Bib::Uri.new(content: "www.nist.gov")],
+            ),
+          )
+          [Bib::Copyright.new(owner: [owner], from: from)]
+        end
+
+        # Fetch links.
+        # @param doc [Hash]
+        # @return [Array<Bib::Uri>]
+        def fetch_link(doc)
+          links = []
+          links << Bib::Uri.new(type: "src", content: doc["uri"]) if doc["uri"]
+          if doc["doi"]
+            links << Bib::Uri.new(type: "doi", content: "https://doi.org/#{doc['doi']}")
+          end
+          links
+        end
+
+        # @return [Array<Bib::Place>]
+        def fetch_place
+          [Bib::Place.new(city: "Gaithersburg", region: [Bib::Place::RegionType.new(iso: "MD")])]
+        end
+
+        def fetch_relations_json(hit) # rubocop:disable Metrics/AbcSize
+          doc = hit[:json]
+          relations = doc["supersedes"].map do |r|
+            doc_relation "supersedes", hit[:code], r["uri"]
+          end
+
+          relations + doc["superseded-by"].map do |r|
+            doc_relation "updates", hit[:code], r["uri"]
+          end
+        end
+
+        # @param type [String]
+        # @param ref [String]
+        # @param uri [String]
+        # @return [Nist::Relation]
+        def doc_relation(type, ref, uri, lang = "en", script = "Latn") # rubocop:disable Metrics/MethodLength
+          if type == "supersedes"
+            descr = Bib::LocalizedMarkedUpString.new(content: "supersedes", language: lang, script: script)
+            t = "obsoletes"
+          else t = type
+          end
+          ids = [Docidentifier.new(content: ref, type: "NIST", primary: true)]
+          link = [Bib::Uri.new(type: "src", content: uri)]
+          bib = ItemData.new(formattedref: Bib::Formattedref.new(content: ref), source: link, docidentifier: ids)
+          Relation.new(type: t, description: descr, bibitem: bib)
+        end
+
+        # @param doc [Hash]
+        # @return [Array<Bib::Keyword>]
+        def fetch_keywords(doc)
+          doc["keywords"].map do |kw|
+            text = kw.is_a?(String) ? kw : kw.text
+            Bib::Keyword.new(vocab: Bib::LocalizedString.new(content: text))
+          end
+        end
+
+        # @param json [Hash]
+        # @return [Nist::CommentPeriod, NilClass]
+        def fetch_commentperiod_json(json)
+          return unless json["comment-from"]
+
+          CommentPeriod.new from: json["comment-from"], to: json["comment-to"]
+        end
+      end
+    end
+  end
+end

data/lib/relaton/nist/series.yaml

@@ -0,0 +1,49 @@
+---
+AMS: Advanced Manufacturing Series
+BRPD: Basic Radio Propagation Predictions Series
+BH: Building and Housing Reports
+BMS: Building Materials and Structures Reports
+BSS: Building Science Series
+CRPL: CRPL Solar-Geophysical Data
+IP: CRPL Ionospheric Predictions
+CIRC: Circulars
+CIS: Consumer Information Series
+CS: Commercial Standards
+CSM: Commercial Standards Monthly
+CSWP: Cybersecurity White Papers
+EAB: Economic Analysis Briefs
+FIPS: Federal Information Processing Standards Publications
+GCR: Grant/Contractor Reports
+HB: Handbooks
+HR: Hydraulic Research in the United States
+IRPL: Interservice Radio Propagation Laboratory
+LCIRC: Letter Circular
+MONO: Monographs
+MP: Miscellaneous Publications
+NCSTAR: National Construction Safety Team Act Reports
+NSRDS: National Standard Reference Data Series
+IR: IR (Interagency/Internal Reports)
+OWMWP: Office of Weights and Measures White Papers
+PC: Photographic Circulars
+RPT: NBS Reports
+SIBS: Special Interior Ballistics Studies
+SP: Special Publications (General)
+SP250: 'SP 250: Calibration Services'
+SP260: 'SP 260: Standard Reference Materials'
+SP300: 'SP 300: Precision Measurement and Calibration'
+SP400: 'SP 400: Semiconductor Measurement Technology'
+SP480: 'SP 480: Law Enforcement Technology'
+SP500: 'SP 500: Computer Systems Technology'
+SP700: 'SP 700: Industrial Measurement Series'
+SP800: 'SP 800: Computer Security Series'
+SP823: 'SP 823: Integrated Services Digital Network Series'
+SP960: 'SP 960: NIST Recommended Practice Guides'
+SP1200: 'SP 1200: Protocols'
+SP1500: 'SP 1500: Working Group Papers'
+SP1800: 'SP 1800: NIST Cybersecurity Practice Guides'
+SP1900: 'SP 1900: Cyber-Physical Systems'
+SP2000: 'SP 2000: Standards Coordination'
+SP2100: 'SP 2100: Conference Proceedings'
+TIBM: Technical Information on Building Materials
+TN: Technical Notes
+TTB: Technology Transfer Brief

data/lib/relaton/nist/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Nist
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-nist".freeze
+    end
+  end
+end

data/lib/relaton/nist.rb

@@ -0,0 +1,34 @@
+require "pubid"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "nist/util"
+require_relative "nist/item_data"
+require_relative "nist/item"
+require_relative "nist/relation"
+require_relative "nist/bibitem"
+require_relative "nist/bibdata"
+require_relative "nist/pubs_export"
+require_relative "nist/hit"
+require_relative "nist/hit_collection"
+require_relative "nist/scraper"
+require_relative "nist/bibliography"
+# require_relative "nist/tech_pubs_parser"
+
+module Relaton
+  module Nist
+    # Pubid-based index (index-v2): `:id` is a Pubid::Nist::Identifier hash, so
+    # search narrows by number via binary search. The legacy string index-v1
+    # (for older gem versions) is rebuilt separately by the relaton-data-nist
+    # crawler, not by this gem's DataFetcher.
+    INDEXFILE = "index-v2"
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/oasis/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/oasis/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Oasis
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/oasis/bibliography.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module Relaton
+  module Oasis
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-oasis/refs/heads/v2/"
+
+      # The publisher token every OASIS printed id starts with, and the one
+      # thing `Pubid::Oasis`'s grammar requires. See #parse_ref.
+      PREFIX = "OASIS "
+
+      # The components a reference may omit. `number` (the specification name)
+      # is deliberately absent — see #ignored.
+      OPTIONAL = %i[version stage part label].freeze
+
+      class << self
+        def search(text, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength
+          Util.info "Fetching from Relaton repository ...", key: text
+          row = find_index_entry(text)
+          unless row
+            Util.info "Not found.", key: text
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          parse_item(fetch_yaml(uri), text)
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Net::ReadTimeout,
+               OpenSSL::SSL::SSLError, Errno::ETIMEDOUT => e
+          raise Relaton::RequestError,
+                "Could not access #{uri}: #{e.message}"
+        end
+
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        #
+        # The pubid `index-v2`. `pubid_class:` is what makes `Relaton::Index`
+        # deserialize the rows into identifiers, sort them by
+        # `id.root.number`, and let `Type#search` bsearch; `file:` names the
+        # local cache, and `Processor#remove_index_file` must name the same
+        # one, or `Db#clear` leaves this cache in place.
+        #
+        # @return [Relaton::Index::Type]
+        #
+        def index
+          Relaton::Index.find_or_create(
+            :oasis, url: "#{ENDPOINT}#{INDEXFILE}.zip",
+            file: "#{INDEXFILE}.yaml",
+            pubid_class: ::Pubid::Oasis::Identifier
+          )
+        end
+
+        #
+        # Find the row that best answers a reference.
+        #
+        # **Pass the pubid, not the string.** `Type#search_candidates` narrows
+        # only when the argument is not a `String`, so passing the reference
+        # text would disable the binary search however the index was built.
+        #
+        # Selection is `Type#search`'s default with no block — pubid's
+        # asymmetric subset match (`Pubid::SubsetMatch`): the reference on the
+        # left, and a component it omits (`version`, `stage`, `part`, `label`)
+        # matches any value, which is what `#ranking_key` then orders.
+        #
+        # A reference pubid cannot parse finds nothing, and there is no
+        # substring fallback. `#parse_ref` supplies the publisher token, so
+        # after normalization the only unparseable inputs are a blank string
+        # and one over pubid's 1000-character cap — and a substring scan for
+        # `""` matches every row in the index, which is worse than a miss.
+        # This is also the v1 -> v2 semantic change IANA recorded: v1 matched
+        # by substring, so `OASIS amqp` resolved to some `amqp-core` record;
+        # v2 matches identifiers, so a partial name no longer resolves.
+        #
+        # @param text [String] the reference as the caller wrote it
+        # @return [Hash, nil] the index row, or nil
+        #
+        def find_index_entry(text)
+          pubid = parse_ref text
+          return unless pubid
+
+          rows = index.search(pubid)
+          rows.max_by { |row| ranking_key pubid, row[:id] }
+        end
+
+        #
+        # An OASIS printed id carries the publisher token, and pubid rejects a
+        # bare slug — but a caller may well write one ("mqtt-v5.0"), and
+        # `Db#fetch` hands the reference through verbatim. Supplying the token
+        # is normalization, not identification, so it happens here rather than
+        # in the grammar.
+        #
+        # The token is stripped **case-insensitively** and re-added in its
+        # canonical form. pubid's grammar matches the literal `OASIS `, so a
+        # lowercase `oasis stix` would otherwise keep its own token, be given a
+        # second one, and search for the specification named `oasis stix`.
+        # Everything after the token stays verbatim: an OASIS slug is
+        # case-sensitive (`STIX`, `amqp-core`, `OpenDocument`).
+        #
+        # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+        # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+        # `"..." is not a recognized standards identifier`
+        # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
+        # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
+        # Rescuing here would collapse "this identifier is malformed" into "no
+        # such document", leaving a caller unable to tell them apart.
+        #
+        # @param text [String]
+        # @return [Pubid::Oasis::Identifier]
+        # @raise [Pubid::Errors::ParseError]
+        #
+        def parse_ref(text)
+          slug = text.to_s.strip.sub(/\A#{Regexp.escape PREFIX}/i, "")
+          ::Pubid::Oasis::Identifier.parse "#{PREFIX}#{slug}"
+        end
+
+        #
+        # The components the reference left out, which the row may therefore
+        # carry freely. `#find_index_entry` no longer needs this list — pubid's
+        # `===` reads the omission from the reference itself — but
+        # `#ranking_key` still counts the components the caller did not ask for.
+        # `number` — the specification name — is never optional: it is the whole
+        # identity of an OASIS record and the key the index bsearches on.
+        #
+        # @param pubid [Pubid::Oasis::Identifier]
+        # @return [Array<Symbol>]
+        #
+        def ignored(pubid)
+          OPTIONAL.select { |attr| pubid.public_send(attr).nil? }
+        end
+
+        #
+        # Order the matched rows. Ignoring a component means "don't care", so a
+        # loose reference matches the more specific rows too and this key
+        # decides between them:
+        #
+        # 1. an **exact printed id** always wins. Five published records are a
+        #    bare specification name that also has versioned siblings — `OASIS
+        #    EDXL`, `OData`, `OSLC`, `SAML`, `WSS` — and each is a real
+        #    document in its own right. Without this key the "newest version"
+        #    rule below answers a request for the bare record with its newest
+        #    sibling; three of the five did exactly that. Published ids are
+        #    unique (0 duplicates in 605 rows), so at most one row scores here;
+        # 2. then the **newest version**, segments compared as integers —
+        #    `v2.1` beats `v1.2.1`, and `v10` beats `v9`, which a string
+        #    compare gets wrong;
+        # 3. then the **least specific** row, i.e. the fewest components the
+        #    reference did not ask for — a bare `OASIS STIX` wants `STIX-v2.1`,
+        #    not `STIX-v2.1-CS02`, and `OASIS STIX-v1.2.1-CS01` wants that
+        #    record itself, not its `-Pt3-Core` part;
+        # 4. then the **later stage revision** (`CS02` over `CS01`). Only the
+        #    digits are compared: OASIS stage letters have no ranking this code
+        #    is entitled to invent;
+        # 5. then the printed id, so the result never depends on index order.
+        #
+        # Checked against every published row: all 605 resolve to their own
+        # record, with and without the publisher token.
+        #
+        # @param pubid [Pubid::Oasis::Identifier] the parsed reference
+        # @param id [Pubid::Oasis::Identifier] a matched row's id
+        # @return [Array]
+        #
+        def ranking_key(pubid, id)
+          exact = id.to_s == pubid.to_s ? 1 : 0
+          extra = ignored(pubid).count { |attr| id.public_send(attr) }
+          [exact, digits(id.version), -extra, digits(id.stage), id.to_s]
+        end
+
+        # Integer segments of a version or stage token, for numeric ordering
+        # ("v1.2.1" -> [1, 2, 1], "CS02" -> [2], nil -> []).
+        #
+        # @param token [String, nil]
+        # @return [Array<Integer>]
+        def digits(token)
+          token.to_s.scan(/\d+/).map(&:to_i)
+        end
+
+        def fetch_yaml(uri)
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError,
+                  "Could not access #{uri}: HTTP #{resp.code}"
+          end
+          resp.body
+        end
+
+        def parse_item(yaml, text)
+          item = Item.from_yaml yaml
+          Util.info "Found: `#{item.docidentifier.first.content}`", key: text
+          item.tap { |i| i.fetched = Date.today.to_s }
+        end
+      end
+    end
+  end
+end

data/lib/relaton/oasis/browser_agent.rb

@@ -0,0 +1,71 @@
+require "ferrum"
+require "nokogiri"
+
+module Relaton
+  module Oasis
+    # Thin Ferrum-backed agent that drives headless Chrome with stealth tweaks
+    # so the Cloudflare-protected oasis-open.org host serves real HTML instead
+    # of a "Just a moment..." challenge. Mirrors the pattern used by
+    # Relaton::Cie::BrowserAgent.
+    class BrowserAgent
+      UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " \
+           "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
+      CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
+      MAX_CHALLENGE_WAIT = 30
+
+      def initialize
+        @browser = Ferrum::Browser.new(
+          headless: true,
+          timeout: 90,
+          process_timeout: 90,
+          window_size: [1366, 768],
+          browser_options: {
+            "disable-blink-features" => "AutomationControlled",
+            "disable-quic" => nil,
+            "no-sandbox" => nil,
+          },
+        )
+        @browser.headers.set(
+          "Accept-Language" => "en-US,en;q=0.9",
+          "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9," \
+                      "image/webp,*/*;q=0.8",
+          "User-Agent" => UA,
+        )
+        @browser.evaluate_on_new_document(<<~JS)
+          Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
+          Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });
+          Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
+          window.chrome = { runtime: {} };
+        JS
+      end
+
+      def get(url)
+        @browser.go_to(url)
+        wait_for_challenge
+        Nokogiri::HTML(@browser.body)
+      end
+
+      # HTTP status code of the most recent navigation's main resource,
+      # or nil if no navigation has happened yet.
+      def last_status
+        @browser&.network&.status
+      end
+
+      def quit
+        @browser&.quit
+      ensure
+        @browser = nil
+      end
+
+      private
+
+      def wait_for_challenge
+        MAX_CHALLENGE_WAIT.times do
+          return unless CHALLENGE_MARKERS.any? { |m| @browser.body.include?(m) }
+
+          sleep 1
+        end
+      end
+    end
+  end
+end