relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/cen/scraper.rb

@@ -75,12 +75,20 @@ module Relaton
           Bib::Status.new(stage: stage)
         end
 
-        # @param hit [RelatonCen::Hit]
+        # The number and part come from the pubid. `#root` is the accessor that
+        # answers for every form: an adopted norm keeps both on the adopted ISO
+        # document, and a supplement keeps both on the document it supplements.
+        # pubid holds a sub-part inside `part` (`61375-2-3` gives `"2-3"`), so
+        # the separator swap reproduces the old `"2:3"`.
+        #
+        # @param hit [Relaton::Cen::Hit]
         # @return [Relaton::Bib::StructuredIdentifier]
         def fetch_structuredid(hit)
-          %r{(?<docnum>\d+)(?:-(?<part>\d+))?(?:-(?<subpart>\d+))?} =~ hit[:code]
-          partnumber = [part, subpart].compact.join(":")
-          StructuredIdentifier.new(docnumber: docnum, partnumber: partnumber, agency: ["CEN"])
+          root = hit.pubid&.root
+          docnumber = root&.number&.to_s
+          partnumber = root&.part.to_s.split("-").join(":")
+          StructuredIdentifier.new(docnumber: docnumber, partnumber: partnumber,
+                                   agency: ["CEN"])
         end
 
         # Fetch relations.
@@ -201,7 +209,7 @@ module Relaton
             doctype: Bib::Doctype.new(content: "international-standard"),
             flavor: "cen",
             ics: fetch_ics(doc),
-            structuredidentifier: fetch_structuredid(hit.hit),
+            structuredidentifier: fetch_structuredid(hit),
           )
         end
       end

data/lib/relaton/cen.rb

@@ -4,6 +4,7 @@ require "digest/md5"
 require "mechanize"
 require "relaton/core"
 require "isoics"
+require "pubid"
 require "relaton/bib"
 require_relative "version"
 require_relative "cen/util"

data/lib/relaton/cie/data_fetcher.rb

@@ -4,6 +4,7 @@ require "English"
 require "fileutils"
 require "ferrum"
 require "nokogiri"
+require "pubid"
 require "relaton/index"
 require "relaton/bib"
 require "relaton/core/data_fetcher"
@@ -21,6 +22,12 @@ module Relaton
       CHALLENGE_MARKERS = ["Just a moment", "challenge-platform"].freeze
       MAX_CHALLENGE_WAIT = 30
 
+      # Raised when the Cloudflare challenge ("Just a moment…") is still showing
+      # after MAX_CHALLENGE_WAIT seconds. It's a retriable signal (see
+      # DataFetcher::RETRIABLE_ERRORS): rather than parse the challenge HTML as a
+      # document, the caller (#time_req) backs the worker off and retries.
+      class ChallengeError < StandardError; end
+
       def initialize
         @browser = Ferrum::Browser.new(
           headless: true,
@@ -68,18 +75,106 @@ module Relaton
 
           sleep 1
         end
+        raise ChallengeError, "Cloudflare challenge did not clear after #{MAX_CHALLENGE_WAIT}s"
       end
     end
 
     class DataFetcher < Relaton::Core::DataFetcher
       URL = "https://www.techstreet.com/cie/searches/31156444?page=1&per_page=100"
 
+      # Default worker-pool size for the parallel detail-fetch phase. Kept small
+      # so N stealth Chrome instances don't burst requests fast enough to trip
+      # Cloudflare; tune via RELATON_CIE_CONCURRENCY (see .concurrency).
+      DEFAULT_CONCURRENCY = 5
+
+      # Per-worker pacing bounds (seconds). Each worker starts at BASE_GAP and
+      # doubles its own gap up to MAX_GAP on trouble (see Pacing / #time_req).
+      BASE_GAP = 1
+      MAX_GAP = 32
+
+      # Number of detail-fetch worker threads. Tunable via env var so
+      # relaton-data-cie's crawler workflow can dial it up for speed or down to
+      # lighten load on techstreet without a code change. Never below 1.
+      def self.concurrency
+        [(ENV["RELATON_CIE_CONCURRENCY"] || DEFAULT_CONCURRENCY).to_i, 1].max
+      end
+
+      def initialize(output, format)
+        super
+        # Guards the shared bookkeeping (@files, @errors, @seen) and the index
+        # mutation while workers fetch detail pages in parallel. The slow
+        # agent.get runs outside this lock; only the cheap build+write is held.
+        @mutex = Mutex.new
+        # output file => catalogue position of the hit that last claimed it, so a
+        # duplicate primary id resolves to the same last-by-position winner the
+        # serial crawl would pick, regardless of worker completion order.
+        @seen = {}
+      end
+
+      # Factory for a fresh stealth browser agent. The pool builds one per worker
+      # (each its own Chrome, each with the full UA/header/`navigator` masking);
+      # #agent memoizes a single one for the serial listing phase. Specs stub
+      # this so one double can back every worker.
+      def build_agent
+        BrowserAgent.new
+      end
+
       def agent
-        @agent ||= BrowserAgent.new
+        @agent ||= build_agent
+      end
+
+      # Per-worker adaptive request pacing. Replaces the old single global 4 s
+      # gap: each worker owns one Pacing, starts at BASE_GAP, and doubles its gap
+      # (capped at MAX_GAP) whenever a request hits trouble (a Cloudflare
+      # challenge or a Ferrum/socket error), so a struggling worker slows itself
+      # without throttling the healthy ones.
+      class Pacing
+        def initialize(base: BASE_GAP, max: MAX_GAP)
+          @base = base
+          @max = max
+          @gap = base
+          @last = nil
+        end
+
+        attr_reader :gap
+
+        # Sleep off whatever remains of this worker's gap since its last request,
+        # then stamp the clock (in ensure, mirroring the old #time_req).
+        def throttle
+          sleep [@gap - (Time.now - @last), 0].max if @last
+        ensure
+          @last = Time.now
+        end
+
+        # Exponential backoff, capped at @max. Called before a retry on trouble.
+        def backoff
+          @gap = [@gap * 2, @max].min
+        end
       end
 
       def index
-        @index ||= Index.find_or_create :cie, file: "index-v1.yaml"
+        @index ||= Index.find_or_create :cie, file: "#{INDEXFILE}.yaml",
+                                              pubid_class: ::Pubid::Cie::Identifier
+      end
+
+      # Parse a docidentifier string into a Pubid::Cie::Identifier, or nil if
+      # pubid can't parse it or it won't survive the structured index — so a
+      # single bad id never aborts the crawl or corrupts index-v2. The caller
+      # (#write_file) records the skip.
+      #
+      # The guard mirrors the read side's acceptance test
+      # (Index::FileIO#id_supported?, `from_hash(to_hash) == to_hash`): the
+      # loader raises InvalidIndexError and rejects the WHOLE index on the first
+      # id that doesn't round-trip, so an id that serializes but can't be
+      # deserialized back must be dropped here rather than poison every lookup.
+      def pubid(id)
+        pid = ::Pubid::Cie.parse id
+        hash = pid.to_hash
+        return nil unless ::Pubid::Cie::Identifier.from_hash(hash).to_hash == hash
+
+        pid
+      rescue StandardError
+        nil
       end
 
       def log_error(msg)
@@ -273,9 +368,24 @@ module Relaton
       end
 
       # @param bib [RelatonCie::BibliographicItem]
-      def write_file(bib) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      # @param pos [Integer, nil] the hit's catalogue position, used only by the
+      #   parallel pool to keep a deterministic last-by-position winner when two
+      #   hits map to the same output file. nil (direct calls) keeps the
+      #   historical warn-and-overwrite behavior.
+      def write_file(bib, pos = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         id = bib.docidentifier[0].content
         file = output_file id
+        # Index every distinct id (the serial crawl indexes each one, even two
+        # that resolve to the same file), before the content-dedup gate below.
+        pid = pubid id
+        if pid
+          index.add_or_update pid, file
+        else
+          Util.warn { "Unparseable id `#{id}` was not indexed (#{file})" }
+        end
+        # Only the last-by-position hit writes the shared file's content.
+        return if superseded? file, pos
+
         if @files.include? file
           Util.warn do
             "File #{file} exists. Docid: #{bib.docidentifier[0].content}\n" \
@@ -283,71 +393,146 @@ module Relaton
           end
         else @files << file
         end
-        index.add_or_update bib.docidentifier[0].content, file
         File.write file, serialize(bib), encoding: "UTF-8"
       end
 
+      # True when another hit with a *later* catalogue position has already
+      # claimed `file`. The serial crawl is last-writer-wins, so that later hit
+      # is canonical and this (earlier or duplicate) content write must be
+      # dropped — making the output independent of worker completion order. With
+      # pos nil there's no position to compare, so nothing is superseded.
+      def superseded?(file, pos)
+        return false if pos.nil?
+
+        prev = @seen[file]
+        return true if prev && prev >= pos
+
+        @seen[file] = pos
+        false
+      end
+
       def to_xml(bib) = bib.to_xml(bibdata: true)
       def to_yaml(bib) = bib.to_yaml
       def to_bibxml(bib) = bib.to_rfcxml
 
+      # Fetch and store one document. The slow detail-page load
+      # (`worker_agent.get`, throttled by this worker's `pacing`) runs outside
+      # the lock — that's the parallel win; the cheap build + write is done under
+      # @mutex so @files/@errors/@seen and the index stay consistent across
+      # workers. Called directly (single stubbed agent) by specs and from the
+      # pool with a per-worker agent/pacing/position.
+      #
       # @param hit [Nokogiri::HTML::Element]
-      def parse_page(hit) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      def parse_page(hit, pos = nil, worker_agent = agent, pacing = nil)
         url = hit.at('h3/a')[:href]
-        doc = time_req { agent.get url }
-        item = ItemData.new(
-          type: "standard", source: fetch_source(url), docnumber: fetch_docnumber(hit),
-          docidentifier: fetch_docid(hit, doc), title: fetch_title(doc),
-          abstract: fetch_abstract(doc), date: fetch_date(doc),
-          edition: fetch_edition(doc), contributor: fetch_contributor(doc),
-          relation: fetch_relation(doc), language: "en", script: "Latn",
-          ext: fetch_ext
-        )
-        write_file item
+        doc = time_req(pacing) { worker_agent.get url }
+        @mutex.synchronize do
+          item = ItemData.new(
+            type: "standard", source: fetch_source(url), docnumber: fetch_docnumber(hit),
+            docidentifier: fetch_docid(hit, doc), title: fetch_title(doc),
+            abstract: fetch_abstract(doc), date: fetch_date(doc),
+            edition: fetch_edition(doc), contributor: fetch_contributor(doc),
+            relation: fetch_relation(doc), language: "en", script: "Latn",
+            ext: fetch_ext
+          )
+          write_file item, pos
+        end
       rescue StandardError => e
         Util.error do
           "Document: #{url}\n#{e.message}\n#{e.backtrace}"
         end
       end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
       def fetch(_source = nil)
-        fetch_doc
+        process_hits collect_hits
+        index.save
         report_errors
       ensure
         @agent&.quit
       end
 
-      def fetch_doc(url = URL)
-        result = time_req { agent.get url }
-        result.xpath("//li[@data-product]").each { |hit| parse_page hit }
+      # Phase 1 (serial): walk the search-result pages, following the "next"
+      # link, and collect every //li[@data-product] hit into one list (~12 page
+      # loads at per_page=100, so serial pacing here is cheap). The detail-page
+      # fetches happen in #process_hits.
+      def collect_hits(url = URL, hits = [])
+        result = time_req(listing_pacing) { agent.get url }
+        hits.concat result.xpath("//li[@data-product]").to_a
         np = result.at '//a[@class="next_page"]'
-        if np
-          next_href = np[:href]
-          next_url = next_href.start_with?("http") ? next_href : "https://www.techstreet.com#{next_href}"
-          fetch_doc next_url
-        else
-          index.save
+        return hits unless np
+
+        next_href = np[:href]
+        next_url = next_href.start_with?("http") ? next_href : "https://www.techstreet.com#{next_href}"
+        collect_hits next_url, hits
+      end
+
+      # Phase 2 (parallel): fan the collected hits out across a bounded worker
+      # pool. Each worker owns its own stealth browser agent and its own adaptive
+      # pacing. The agents are built up front on this thread so a Chrome-launch
+      # failure aborts fast (before any hit is enqueued and before the index is
+      # saved) rather than leaving a dead worker that can't drain the queue; and
+      # every created agent is quit in the ensure even if enqueuing or a worker
+      # raises. Hits carry their catalogue position so #write_file keeps a
+      # deterministic winner for any duplicate output file — byte-identical
+      # output regardless of completion order.
+      def process_hits(hits)
+        return if hits.empty?
+
+        n = self.class.concurrency
+        agents = []
+        queue = SizedQueue.new(n * 2)
+        n.times { agents << build_agent }
+        workers = agents.map { |worker_agent| spawn_worker(queue, worker_agent) }
+        hits.each_with_index { |hit, pos| queue << [hit, pos] }
+        n.times { queue << nil } # poison pills
+        workers.each(&:join)
+      ensure
+        agents&.each(&:quit)
+      end
+
+      # One pool worker: drains the queue with its own browser agent + pacing
+      # until the poison pill (nil). parse_page swallows per-document errors, so
+      # the thread runs to the pill; the agent is quit by #process_hits.
+      def spawn_worker(queue, worker_agent)
+        pacing = Pacing.new
+        Thread.new do
+          while (item = queue.pop)
+            hit, pos = item
+            parse_page hit, pos, worker_agent, pacing
+          end
         end
       end
 
+      # Pacing for the serial listing phase (Phase 1). Also the fallback pacing
+      # for #time_req when no per-worker pacing is passed (direct calls / specs).
+      def listing_pacing
+        @listing_pacing ||= Pacing.new
+      end
+
       RETRIABLE_ERRORS = [
         SocketError,
         Ferrum::TimeoutError,
         Ferrum::PendingConnectionsError,
-        Ferrum::StatusError
+        Ferrum::StatusError,
+        BrowserAgent::ChallengeError
       ].freeze
 
-      def time_req
+      # Run a throttled request through the given worker's pacing, retrying up to
+      # 4 times on a retriable error (challenge / Ferrum / socket) and backing
+      # that worker's gap off before each retry.
+      def time_req(pacing = listing_pacing)
+        pacing ||= listing_pacing
         tries = 0
         begin
           tries += 1
-          sleep [4 - (Time.now - @last_request_time).to_i, 0].max if @last_request_time
+          pacing.throttle
           yield
         rescue *RETRIABLE_ERRORS => e
+          pacing.backoff
           retry if tries < 4
           raise e
-        ensure
-          @last_request_time = Time.now
         end
       end
     end

data/lib/relaton/cie/processor.rb

@@ -59,7 +59,9 @@ module Relaton
       #
       def remove_index_file
         require_relative "../cie"
-        Relaton::Index.find_or_create(:cie, url: true, file: "#{INDEXFILE}.yaml").remove_file
+        Relaton::Index.find_or_create(
+          :cie, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
       end
     end
   end

data/lib/relaton/cie/scrapper.rb

@@ -9,8 +9,21 @@ module Relaton
         # @param code [String]
         # @return [Relaton::Cie::ItemData]
         def scrape_page(code)
-          index = Index.find_or_create :cie, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml"
-          row = index.search(code).min_by { |r| r[:id] }
+          # An unrecognized reference raises; like ISO and 3GPP we let it
+          # propagate -- relaton-cli rescues Pubid::Errors::Error and renders
+          # "... is not a recognized standards identifier". Partial refs
+          # (`CIE 001`, `CIE 15`) parse, so nothing valid is lost.
+          pubid = ::Pubid::Cie.parse code
+          index = Index.find_or_create :cie, url: "#{ENDPOINT}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+                                              pubid_class: ::Pubid::Cie::Identifier
+          # Pass the parsed pubid (not the raw String) so index-v2 narrows
+          # candidates by number via binary search before the block runs; the
+          # block keeps the broad substring match the string index gave.
+          # Rows are Pubid::Cie::Identifier objects (not Comparable), so pick by
+          # the string form.
+          needle = pubid.to_s
+          row = index.search(pubid) { |r| r[:id].to_s.include?(needle) }
+                     .min_by { |r| r[:id].to_s }
           return unless row
 
           parse_page "#{ENDPOINT}#{row[:file]}", code

data/lib/relaton/cie.rb

@@ -1,5 +1,6 @@
 require "nokogiri"
 # require "parslet"
+require "pubid"
 require "relaton/index"
 require "relaton/bib"
 # require "relaton_bib/name_parser"
@@ -14,7 +15,7 @@ require_relative "cie/scrapper"
 
 module Relaton
   module Cie
-    INDEXFILE = "index-v1".freeze
+    INDEXFILE = "index-v2".freeze
 
     # Returns hash of XML reammar
     # @return [String]

data/lib/relaton/core/data_fetcher.rb

@@ -15,6 +15,16 @@ module Relaton
         @format = format
         @ext = format.sub "bibxml", "xml"
         @files = Set.new
+        # path => docid that reserved it, for #unique_output_file. Distinct
+        # from @files, which flavors use for their own duplicate handling:
+        # this one has to know WHICH document owns a path, not just that it
+        # is taken.
+        @file_docids = {}
+        # Paths this process wrote during this run; see #write_unique.
+        @written = Set.new
+        # Set true by a fetcher that writes from forked worker processes,
+        # where @file_docids cannot see a peer's claim. See #write_unique.
+        @cross_process = false
         # @docs = []
         @errors = Hash.new(true)
       end
@@ -55,7 +65,14 @@ module Relaton
 
       def report_errors
         gh_issue # register the channel before logging
-        @errors.select { |_, v| v }.each_key { |k| log_error "Failed to fetch #{k}" }
+        @errors.select { |_, v| v }.each do |key, value|
+          # A String value IS the message: a specific, per-document failure
+          # such as an unparseable identifier, reported through this same
+          # channel without needing a per-flavor override. A boolean means
+          # "this field failed for every record" — what the flavors' own
+          # ERROR_KEYS track — and its message is derived from the key.
+          log_error value.is_a?(String) ? value : "Failed to fetch #{key}"
+        end
         @gh_issue&.create_issue
       end
 
@@ -63,11 +80,111 @@ module Relaton
         raise NoMatchingPatternError, "#{self.class}#log_error method must be implemented"
       end
 
+      # Most filesystems cap a single path component at 255 bytes.
+      MAX_BASENAME_BYTES = 255
+
+      # Create-or-fail. The failure is the point: it is how one process learns
+      # that another already holds a path. See #write_unique.
+      EXCLUSIVE = File::WRONLY | File::CREAT | File::EXCL
+
       # @param [String] document ID
       # @return [String] filename based on PubID identifier
+      #
+      # A docid can be pathologically long (pubid's `to_s` for amendment docs
+      # embeds the full "(Amendment to … as amended by …)" clause), which would
+      # make the basename exceed the OS limit and raise Errno::ENAMETOOLONG on
+      # write. When that happens, truncate the sanitized id and append a short
+      # digest of the full docid so the filename stays bounded, unique, and
+      # deterministic (every call site round-trips through this method).
       def output_file(docid)
-        id = docid.downcase.gsub(/[.\s\/:()-]+/, "-").delete_suffix("-")
-        File.join @output, "#{id}.#{@ext}"
+        id = docid.downcase.gsub(/[.,\s\/:()-]+/, "-").delete_suffix("-")
+        ext = ".#{@ext}"
+        limit = MAX_BASENAME_BYTES - ext.bytesize
+        if id.bytesize > limit
+          require "digest"
+          suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
+          # `id` has no consecutive "-" (gsub collapsed runs above), so
+          # truncation leaves at most one trailing "-" — delete_suffix is
+          # enough and avoids a polynomial-ReDoS regex on the docid.
+          id = id.byteslice(0, limit - suffix.bytesize).scrub("").delete_suffix("-") + suffix
+        end
+        File.join @output, "#{id}#{ext}"
+      end
+
+      #
+      # Reserve a unique output path for `docid`.
+      #
+      # `output_file` sanitizes ".", ",", "/", ":", "(", ")", "-" and whitespace
+      # all to "-", so two DISTINCT docids can map to one path — live instance in
+      # relaton-data-iana: `rpki/signed-objects` and `rpki-signed-objects` both
+      # give `data/rpki-signed-objects.yaml`. A caller that merely warns and
+      # writes anyway leaves one file holding the wrong document for one of two
+      # index ids: a wrong answer, not a missing one.
+      #
+      # Returns `output_file(docid)` when that path is free, or when it is
+      # already held by this SAME docid — a genuine duplicate is the caller's
+      # business (skip / merge / last-wins), and this method must not turn one
+      # into two files. Only a real clash with a DIFFERENT docid gets a variant,
+      # suffixed with a digest of this docid so the name depends on the document
+      # and not on encounter order: adding a record never renames an existing
+      # file. (Which member of a clashing pair keeps the plain path does follow
+      # write order, which is stable for a given corpus.)
+      #
+      # @param [String] docid
+      # @return [String] path, reserved for this docid
+      #
+      def unique_output_file(docid)
+        file = output_file docid
+        owner = @file_docids[file]
+        file = digest_output_file(docid) unless owner.nil? || owner == docid
+        reserve file, docid
+      end
+
+      #
+      # Write `content` for `docid`, never clobbering a different document.
+      #
+      # `output_file` is not injective, so the plain path may belong to someone
+      # else. `unique_output_file` settles that within this process; the file is
+      # then created with `O_EXCL` so a peer *process* cannot be clobbered
+      # either. Returns the path actually written.
+      #
+      # The four outcomes, each load-bearing:
+      #
+      # * **We already wrote this path for this docid** — plain overwrite, so a
+      #   genuine duplicate stays one file and the flavor's own duplicate
+      #   handling (skip / merge / last-wins) still decides what happens.
+      # * **`O_EXCL` succeeds** — the normal case.
+      # * **`EEXIST`, single process** — `unique_output_file` has already proved
+      #   that no peer of ours holds this path, so the file can only be a
+      #   leftover from an earlier crawl. Overwrite it. Every single-process
+      #   flavor takes this branch, and it involves no guesswork.
+      # * **`EEXIST`, `@cross_process`** — a peer's file and a leftover are
+      #   indistinguishable, so take a path of our own rather than risk
+      #   destroying a record. The caller reconciles the names afterwards, in
+      #   the parent, where it can see every docid (see
+      #   `Relaton::Ietf::DataFetcher#reconcile_output_files`).
+      #
+      # Deliberately NOT a wall-clock "is this file older than the crawl?" test.
+      # A crawl runs into a populated `data/`, so `EEXIST` is the common case on
+      # a re-run and such a test would decide it for every record; its
+      # false-"stale" direction is the silent overwrite this method exists to
+      # prevent, and coarse filesystem mtime granularity makes that reachable.
+      #
+      # @param [String] docid
+      # @param [String] content serialized document
+      # @return [String] path written
+      #
+      def write_unique(docid, content)
+        file = unique_output_file docid
+        return force_write(file, content) if @written.include?(file)
+
+        File.write file, content, mode: EXCLUSIVE, encoding: "UTF-8"
+        @written << file
+        file
+      rescue Errno::EEXIST
+        return force_write(file, content) unless @cross_process
+
+        force_write reserve(digest_output_file(docid), docid), content
       end
 
       #
@@ -92,6 +209,36 @@ module Relaton
       def to_bibxml(bib)
         raise NotImplementedError, "#{self.class}#to_bibxml method must be implemented"
       end
+
+      private
+
+      # Write, clobbering whatever is there. The caller has already established
+      # that the path is ours to take.
+      def force_write(file, content)
+        File.write file, content, encoding: "UTF-8"
+        @written << file
+        file
+      end
+
+      # Record `docid` as the owner of `file`, unless someone got there first,
+      # and return `file`. The reservation table must never hand one path to two
+      # documents; see #unique_output_file.
+      def reserve(file, docid)
+        @file_docids[file] ||= docid
+        file
+      end
+
+      # `output_file`'s path with a docid digest appended to the basename, kept
+      # inside the same byte cap. Deterministic: same docid, same name.
+      def digest_output_file(docid)
+        require "digest"
+        ext = ".#{@ext}"
+        suffix = "-#{Digest::SHA1.hexdigest(docid)[0, 12]}"
+        stem = File.basename output_file(docid), ext
+        limit = MAX_BASENAME_BYTES - ext.bytesize - suffix.bytesize
+        stem = stem.byteslice(0, limit).scrub("").delete_suffix("-") if stem.bytesize > limit
+        File.join @output, "#{stem}#{suffix}#{ext}"
+      end
     end
   end
 end