relaton 3.0.0.pre.alpha.5 → 3.0.0.pre.alpha.7

data/lib/relaton/db.rb

@@ -63,25 +63,23 @@ module Relaton
     #  or "YYYY-MM-DD")
     #
     # @return [nil, RelatonBib::BibliographicItem, ...]
+    # @raise [Relaton::UnknownReferenceError] no flavor recognizes the
+    #   reference (see Registry#route)
     ##
-    def fetch(text, year = nil, opts = {})
+    def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
       reference = text.strip
-      require "pubid"
-      parsed = begin
-        Pubid.parse(reference)
-      rescue Pubid::Errors::Error, Parslet::ParseFailed
-        nil
-      end
-      stdclass = (parsed && @registry.class_by_pubid(parsed)) ||
-                 @registry.class_by_ref(reference) || return
+      stdclass, pubid = @registry.route(reference)
       processor = @registry[stdclass]
       ref = if processor.respond_to?(:urn_to_code)
               processor.urn_to_code(reference)&.first
             else reference
             end
       ref ||= reference
+      # The routed pubid is the parse of `reference`; a URN rewritten to a
+      # code is parsed again by the flavor.
+      pubid = nil unless ref == reference
       result = combine_doc ref, year, opts, stdclass
-      result || check_bibliocache(ref, year, opts, stdclass)
+      result || fetch_routed(ref, year, opts, stdclass, pubid)
     end
 
     # @see Relaton::Db#fetch
@@ -119,7 +117,7 @@ module Relaton
     #   nil if document not found
     #
     def fetch_async(ref, year = nil, opts = {}, &block) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
-      stdclass = @registry.class_by_ref ref
+      stdclass = async_route ref
       if stdclass
         unless @queues[stdclass]
           processor = @registry[stdclass]
@@ -141,23 +139,32 @@ module Relaton
       end
     end
 
+    #
+    # Fetch with the flavor the caller names, else the routed one.
+    #
+    # @param stdclass [Symbol, String, nil] a processor short name
+    #   (`:relaton_iso`, as relaton-cli passes it) or a prefix (`"ISO"`)
+    # @raise [Relaton::UnknownReferenceError] no flavor is named and none
+    #   recognizes the reference
+    #
     def fetch_std(code, year = nil, stdclass = nil, opts = {})
-      std = nil
-      @registry.processors.each do |name, processor|
-        std = name if processor.prefix == stdclass
-      end
-      std = @registry.class_by_ref(code) or return nil unless std
+      std = named_class(stdclass)
+      # A flavor the caller names is the one asked: no co-publisher.
+      return check_bibliocache(code, year, opts, std) if std
 
-      check_bibliocache(code, year, opts, std)
+      std, pubid = @registry.route(code)
+      fetch_routed(code, year, opts, std, pubid)
     end
 
     # The document identifier class corresponding to the given code
     # @param code [String]
     # @return [Array]
     def docid_type(code)
-      stdclass = @registry.class_by_ref(code) or return [nil, code]
+      stdclass, = @registry.route(code)
       _, code = strip_id_wrapper(code, stdclass)
       [@registry[stdclass].idtype, code]
+    rescue UnknownReferenceError
+      [nil, code]
     end
 
     # @param key [String]
@@ -186,19 +193,42 @@ module Relaton
 
     private
 
-    def fetch_doc(code, year, opts, processor)
-      if Db.configuration.use_api then fetch_api(code, year, opts, processor)
-      else processor.get(code, year, opts)
+    # The flavor routing picks for `fetch_async`'s per-flavor queue; nil
+    # (logged) when no flavor recognizes the reference.
+    def async_route(ref)
+      @registry.route(ref).first
+    rescue UnknownReferenceError => e
+      Util.info e.message, key: ref
+      nil
+    end
+
+    # The processor a caller names by short name (`:relaton_iso`) or prefix.
+    def named_class(stdclass)
+      return unless stdclass
+
+      short = stdclass.to_sym
+      return short if @registry.processors.key?(short)
+
+      @registry.processors.detect { |_, p| p.prefix == stdclass.to_s }&.first
+    end
+
+    # The flavor's `get` receives the parsed pubid when there is one
+    # (relaton#205), else the String. The API request sends the String the
+    # caller wrote.
+    def fetch_doc(code, year, opts, processor, query = nil)
+      if Db.configuration.use_api
+        fetch_api(code, year, opts, processor, query)
+      else processor.get(query || code, year, opts)
       end
     end
 
-    def fetch_api(code, year, opts, processor)
+    def fetch_api(code, year, opts, processor, query = nil)
       url = "#{Db.configuration.api_host}" \
             "/api/v1/document?#{params(code, year, opts)}"
       rsp = Net::HTTP.get_response URI(url)
       processor.from_xml rsp.body if rsp.code == "200"
     rescue Errno::ECONNREFUSED
-      processor.get(code, year, opts)
+      processor.get(query || code, year, opts)
     end
 
     def params(code, year, opts)
@@ -281,15 +311,36 @@ module Relaton
     # that query is not cached. A processor with no pubid class gets the
     # legacy string key.
     #
+    # @param pubid [Pubid::Identifier, nil] the reference as routing parsed
+    #   it, so the processor does not parse it again
     # @return [Pubid::Identifier, String, nil]
     #
-    def cache_key(code, year, opts, stdclass)
+    def cache_key(code, year, opts, stdclass, pubid = nil)
       processor = @registry[stdclass]
       unless processor.pubid_class
         return std_id(code, year, opts, stdclass).first
       end
 
-      processor.cache_key(code, year, opts)
+      processor.cache_key(code, year, opts, pubid)
+    end
+
+    #
+    # The pubid `get` receives, and the cache key folded from it: one parse
+    # of the reference serves both (relaton#205). A processor with no pubid
+    # class gets the String and the legacy string key.
+    #
+    # @param pubid [Pubid::Identifier, nil] routing's parse
+    # @return [Array(Pubid::Identifier, Object)] the query pubid (nil: `get`
+    #   gets the String) and the key (nil: not cached)
+    #
+    def query_and_key(code, year, opts, stdclass, pubid)
+      processor = @registry[stdclass]
+      unless processor.pubid_class
+        return [nil, std_id(code, year, opts, stdclass).first]
+      end
+
+      query = processor.query_pubid(code, opts, pubid)
+      [query, query && processor.cache_key(code, year, opts, query)]
     end
 
     # The key of a fetched document, from its primary identifier. The
@@ -317,53 +368,112 @@ module Relaton
       [prefix, code]
     end
 
-    def bib_retval(entry, stdclass)
-      if entry && !entry.match?(/^not_found/)
-        @registry[stdclass].from_xml(entry)
+    # The routed flavor, then its co-publishers' flavors (relaton#205).
+    def fetch_routed(code, year, opts, stdclass, pubid)
+      check_bibliocache(code, year, opts, stdclass, pubid: pubid) ||
+        copublisher_fetch(code, year, opts, stdclass, pubid)
+    end
+
+    #
+    # A co-published document the lead flavor's catalog does not have:
+    # ask the co-publishers' flavors, in the order the parsed pubid holds
+    # them (relaton#205; pubid#469, #472). Each one keys, caches and fetches
+    # with its own flavor, so a second fetch is answered from the caches.
+    # A co-publisher whose flavor gives no pubid for the lead's printed form
+    # is skipped. Only that probe parse is rescued: an error from inside the
+    # co-publisher's fetch propagates.
+    #
+    # @return [Relaton::Bib::ItemData, nil]
+    #
+    def copublisher_fetch(code, year, opts, stdclass, pubid)
+      # The reference as the lead read it: no `PREFIX(...)` wrapper, no en dash.
+      _, code = strip_id_wrapper(code, stdclass)
+      copublisher_classes(code, opts, stdclass, pubid).each do |publisher, co|
+        Util.info "Not found; trying co-publisher `#{publisher}`", key: code
+        query = copublisher_query(code, opts, co, publisher) or next
+        bib = check_bibliocache(code, year, opts, co, pubid: query)
+        return bib if bib
+      end
+      nil
+    end
+
+    # @return [Array<Array(String, Symbol)>] each co-publisher that a flavor
+    #   other than the lead serves, with that flavor
+    def copublisher_classes(code, opts, stdclass, pubid)
+      processor = @registry[stdclass]
+      query = processor.query_pubid(code, opts, pubid) or return []
+
+      processor.copublishers(query).filter_map do |publisher|
+        co = @registry.class_by_publisher(publisher)
+        [publisher, co] if co && co != stdclass
+      end.uniq(&:last)
+    end
+
+    # The co-publisher flavor's own parse of the reference, or nil when it
+    # gives none (a parse error, or a flavor that reads it as a miss: IEEE),
+    # which is logged.
+    #
+    # @return [Pubid::Identifier, nil]
+    def copublisher_query(code, opts, stdclass, publisher)
+      query = begin
+        @registry[stdclass].query_pubid(code, opts)
+      rescue ::Pubid::Errors::Error, Parslet::ParseFailed
+        nil
       end
+      return query if query
+
+      Util.info "Not found; co-publisher `#{publisher}` cannot read it",
+                key: code
+      nil
+    end
+
+    def bib_retval(entry, stdclass)
+      @registry[stdclass].from_xml(entry) if entry.is_a?(String)
     end
 
-    def check_bibliocache(code, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
+    def check_bibliocache(code, year, opts, stdclass, pubid: nil) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/MethodLength,Metrics/PerceivedComplexity
       _, searchcode = strip_id_wrapper(code, stdclass)
-      id = cache_key(searchcode, year, opts, stdclass)
+      query, id = query_and_key(searchcode, year, opts, stdclass, pubid)
       db = id && (@local_db || @db)
       altdb = @local_db && @db ? @db : nil
       if db.nil?
         return if opts[:fetch_db]
 
-        bibentry = new_bib_entry(searchcode, year, opts, stdclass)
+        bibentry = new_bib_entry(searchcode, year, opts, stdclass, query: query)
         return bib_retval(bibentry, stdclass)
       end
       if date_range?(opts)
-        return check_date_range(searchcode, id, year, opts, stdclass)
+        return check_date_range(searchcode, id, year, opts, stdclass, query)
       end
 
       @semaphore.synchronize { db.expire id, year }
       if altdb
-        return bib_retval(altdb[id], stdclass) if opts[:fetch_db]
+        return bib_retval(altdb.read(id), stdclass) if opts[:fetch_db]
 
         @semaphore.synchronize do
           db.clone_entry id, altdb if altdb.valid_entry? id, year
         end
-        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
+                      query: query)
         @semaphore.synchronize do
           altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
         end
       else
-        return bib_retval(db[id], stdclass) if opts[:fetch_db]
+        return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
 
-        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id)
+        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
+                      query: query)
       end
-      bib_retval(db[id], stdclass)
+      bib_retval(db.read(id), stdclass)
     end
 
     def new_bib_entry(code, year, opts, stdclass, **args)
-      entry = @semaphore.synchronize { args[:db] && args[:db][args[:id]] }
+      entry = @semaphore.synchronize { args[:db]&.read(args[:id]) }
       if !entry || opts[:no_cache]
         return fetch_entry(code, year, opts, stdclass, **args)
       end
 
-      if entry&.match?(/^not_found/)
+      if entry.is_a?(NotFound)
         Util.info "not found in cache, if you wish to " \
                   "ignore cache please use `no-cache` option.", key: code
         return
@@ -373,15 +483,16 @@ module Relaton
 
     def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
       processor = @registry[stdclass]
-      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1))
+      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
+                      args[:query])
       entry = bib_entry bib
       return entry if args[:db].nil?
 
       # `no_cache` refreshes a cached entry, but a failed fetch does not
-      # replace a cached document with `not_found`.
+      # replace a cached document with a not-found entry.
       refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
       @semaphore.synchronize do
-        if refresh || !args[:db][args[:id]]
+        if refresh || !args[:db].read(args[:id])
           save_bib args[:db], args[:id], bib, entry, stdclass
         end
       end
@@ -404,7 +515,7 @@ module Relaton
     # with the range, and its answer is cached under its own identifier only:
     # the query row keeps pointing to the latest edition.
     #
-    def check_date_range(code, key, year, opts, stdclass) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
+    def check_date_range(code, key, year, opts, stdclass, query = nil) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity
       caches = [@local_db, @db].compact
       cached = caches.flat_map { |c| c.candidates(key) }.filter_map do |_, xml|
         date = published_date(xml)
@@ -414,7 +525,8 @@ module Relaton
       return if opts[:fetch_db]
 
       processor = @registry[stdclass]
-      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1))
+      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
+                      query)
       return unless bib.respond_to?(:to_xml)
 
       entry = bib_entry bib
@@ -427,19 +539,21 @@ module Relaton
       bib_retval entry, stdclass
     end
 
-    def net_retry(code, year, opts, processor, retries)
-      fetch_doc code, year, opts, processor
+    # @param query [Pubid::Identifier, nil] what `get` receives in place of
+    #   the String +code+ (Db#query_and_key)
+    def net_retry(code, year, opts, processor, retries, query = nil)
+      fetch_doc code, year, opts, processor, query
     rescue Relaton::RequestError => e
       raise e unless retries > 1
 
-      net_retry(code, year, opts, processor, retries - 1)
+      net_retry(code, year, opts, processor, retries - 1, query)
     end
 
     def bib_entry(bib)
       if bib.respond_to?(:to_xml)
         bib.to_xml(bibdata: true)
       else
-        "not_found #{Date.today}"
+        NotFound.new(fetched: Date.today.to_s)
       end
     end
 

data/lib/relaton/ecma/bibliography.rb

@@ -43,7 +43,8 @@ module Relaton
         # `ECMA-418` does not match `ECMA-418-1`. The class must be identical,
         # which keeps `ECMA-100` and `ECMA TR/100` apart.
         #
-        # @param ref [String] the ECMA reference (e.g. "ECMA-6", "ECMA-269 ed3 vol2")
+        # @param ref [String, Pubid::Ecma::Identifier] the ECMA reference
+        #   (e.g. "ECMA-6", "ECMA-269 ed3 vol2"), or its parse
         #
         # @return [Array<Hash>] matching index rows
         #
@@ -62,7 +63,7 @@ module Relaton
         # regex accepted, so every reference shape the flavor has to handle
         # parses without normalization here.
         #
-        # @param ref [String]
+        # @param ref [String, Pubid::Ecma::Identifier]
         # @return [Pubid::Ecma::Identifier, nil]
         #
         # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
@@ -73,20 +74,24 @@ module Relaton
         # Rescuing here would collapse "this identifier is malformed" into "no
         # such document", leaving a caller unable to tell them apart.
         def parse_ref(ref)
+          # A pubid from Relaton::Db (relaton#205) is used as it is.
+          return ref unless ref.is_a?(String)
+
           ::Pubid::Ecma::Identifier.parse ref.to_s.strip
         end
 
-        # @param code [String] the ECMA standard Code to look up (e..g "ECMA-6")
+        # @param ref [String, Pubid::Ecma::Identifier] the ECMA reference
+        #   (e.g. "ECMA-6"), or its parse from Relaton::Db (relaton#205)
         # @param year [String] not used
         # @param opts [Hash] not used
         # @return [Relaton::Ecma::ItemData] Relaton of reference
-        def get(code, _year = nil, _opts = {})
-          Util.info "Fetching from Relaton repository ...", key: code
-          result = fetch_doc(code)
+        def get(ref, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: ref.to_s
+          result = fetch_doc(ref)
           if result
-            Util.info "Found: `#{result.docidentifier.first.content}`", key: code
+            Util.info "Found: `#{result.docidentifier.first.content}`", key: ref.to_s
           else
-            Util.info "Not found.", key: code
+            Util.info "Not found.", key: ref.to_s
           end
           result
         end
@@ -106,7 +111,7 @@ module Relaton
         #
         # `r[:file]` breaks the tie, because the index sort is not stable.
         #
-        # @param ref [String]
+        # @param ref [String, Pubid::Ecma::Identifier]
         # @return [Hash, nil]
         #
         def best_match(ref)
@@ -127,8 +132,8 @@ module Relaton
           edition.to_s.split(".").map(&:to_i)
         end
 
-        def fetch_doc(code)
-          row = best_match code
+        def fetch_doc(ref)
+          row = best_match ref
           return unless row
 
           url = "#{ENDPOINT}#{row[:file]}"
@@ -137,11 +142,11 @@ module Relaton
         rescue Mechanize::ResponseCodeError => e
           return if e.response_code == "404"
 
-          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+          raise Relaton::RequestError, "No document found for #{ref} reference. #{e.message}"
         rescue Mechanize::RedirectLimitReachedError, Timeout::Error,
             Mechanize::UnauthorizedError, Mechanize::UnsupportedSchemeError,
             Mechanize::ResponseReadError, Mechanize::ChunkedTerminationError => e
-          raise Relaton::RequestError, "No document found for #{code} reference. #{e.message}"
+          raise Relaton::RequestError, "No document found for #{ref} reference. #{e.message}"
         end
       end
     end

data/lib/relaton/etsi/bibliography.rb

@@ -6,14 +6,15 @@ module Relaton
     module Bibliography
       SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-etsi/refs/heads/v2/"
 
-      # @param text [String]
+      # @param ref [String, ::Pubid::Etsi::Identifier]
       # @return [Relaton::Etsi::ItemData, nil]
-      def search(text) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def search(ref) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
         # An unrecognized reference raises Pubid::Errors::ParseError; like
         # ISO we let it propagate — the CLI turns it into a friendly message
         # and API callers rescue it themselves. Valid partial refs parse with
         # the omitted refinements (version/date/part) left blank.
-        pubid = ::Pubid::Etsi.parse text
+        # A parsed pubid comes from Relaton::Db (relaton#205); it is only read.
+        pubid = ref.is_a?(String) ? ::Pubid::Etsi.parse(ref) : ref
 
         index = Relaton::Index.find_or_create :etsi, url: "#{SOURCE}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
                                                      pubid_class: ::Pubid::Etsi::Identifier
@@ -89,19 +90,19 @@ module Relaton
         [(id.version&.version).to_s.split(".").map(&:to_i), id.date.to_s]
       end
 
-      # @param ref [String] the ETSI standard Code to look up
+      # @param ref [String, ::Pubid::Etsi::Identifier] the ETSI standard Code to look up
       # @param year [String, nil] year
       # @param opts [Hash] options
       # @return [Relaton::Etsi::ItemData, nil]
       def get(ref, _year = nil, _opts = {})
-        Util.info "Fetching from Relaton repository ...", key: ref
+        Util.info "Fetching from Relaton repository ...", key: ref.to_s
         result = search(ref)
         unless result
-          Util.info "Not found.", key: ref
+          Util.info "Not found.", key: ref.to_s
           return
         end
 
-        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref.to_s
         result
       end
 

data/lib/relaton/etsi/data_fetcher.rb

@@ -45,22 +45,117 @@ module Relaton
         report_errors
       end
 
-      def fetch_remaining_pages(first_page)
+      #
+      # Fetch pages 2..N. A page that stays bad after its retries is fetched
+      # again after the last page; if it is still bad, the crawl fails. A page
+      # is never skipped: relaton-data-etsi deletes data/ before the crawl and
+      # commits what it writes, so a skipped page would unpublish its documents.
+      #
+      # N comes from page 1's total_count, but the result set is sorted by
+      # deliverable number, so a document published during the crawl shifts the
+      # later pages by one. The crawl therefore reads on past N while the pages
+      # stay full (at most MAX_EXTRA_PAGES), and one page past a deferred last
+      # page; a record read twice overwrites its own file.
+      #
+      def fetch_remaining_pages(first_page) # rubocop:disable Metrics/MethodLength
+        total_pages = last = last_page(first_page)
+        deferred = []
+        page = 2
+        while page <= last
+          check_extra_pages page, total_pages
+          size = fetch_next_page(page, deferred)
+          break if size&.zero?
+
+          last = page + 1 if page == last && read_on?(size, page, total_pages)
+          page += 1
+        end
+        fetch_deferred_pages(deferred, total_pages)
+      end
+
+      def last_page(first_page)
         total = first_page.first ? first_page.first["total_count"].to_i : 0
-        total_pages = (total / PAGE_SIZE.to_f).ceil
-        (2..total_pages).each do |page|
-          records = fetch_page(page)
-          break if records.empty?
+        last = (total / PAGE_SIZE.to_f).ceil
+        first_page.size >= PAGE_SIZE ? [last, 2].max : last
+      end
+
+      #
+      # Whether to read the page after the last one: after a full page, or
+      # after a deferred page inside the total_count range. A deferred page
+      # past that range does not extend it, so a server that answers bad
+      # bodies past the end cannot keep the crawl going.
+      #
+      def read_on?(size, page, total_pages)
+        size ? size >= PAGE_SIZE : page <= total_pages
+      end
+
+      def check_extra_pages(page, total_pages)
+        return if page <= total_pages + MAX_EXTRA_PAGES
+
+        raise BadPage, "ETSI page #{page} is still full #{MAX_EXTRA_PAGES} " \
+                       "pages past total_count (#{total_pages} pages)."
+      end
+
+      #
+      # @return [Integer, nil] the number of records on the page; nil for a
+      #   deferred page
+      #
+      def fetch_next_page(page, deferred)
+        records = fetch_page(page)
+        process_records(records)
+        records.size
+      rescue BadPage => e
+        Util.warn "#{e.message} Fetching it again after the last page."
+        deferred << page
+        nil
+      end
 
-          process_records(records)
+      def fetch_deferred_pages(pages, total_pages)
+        return if pages.empty?
+
+        sleep DEFERRED_DELAY
+        failed = pages.filter_map do |page|
+          refetch_page page, total_pages
+        rescue BadPage => e
+          e.message
         end
+        raise BadPage, failed.join(" ") if failed.any?
+      end
+
+      # @return [String, nil] the failure message, nil on success
+      def refetch_page(page, total_pages)
+        records = fetch_page(page)
+        if records.empty? && page <= total_pages
+          return "ETSI page #{page} is empty on the re-fetch."
+        end
+
+        process_records records
+        nil
       end
 
       def fetch_page(page)
         date = Time.now.to_date + 1
         timestamp = (Time.now.to_f * 1000).to_i
         url = format(SOURCEURL, page: page, date: date, timestamp: timestamp)
-        JSON.parse(fetch_with_retry(url))
+        fetch_with_retry(url) { |body| parse_page(page, body) }
+      end
+
+      #
+      # ETSI's data.php sometimes answers with PHP's `Array` text in place of
+      # JSON (relaton-data-etsi crawl 36761804954); a re-fetch gets JSON.
+      #
+      # @raise [BadPage] when the body is not a JSON array
+      #
+      def parse_page(page, body)
+        records = JSON.parse(body)
+        return records if records.is_a?(Array)
+
+        raise BadPage, bad_page_message(page, body)
+      rescue JSON::ParserError
+        raise BadPage, bad_page_message(page, body)
+      end
+
+      def bad_page_message(page, body)
+        "ETSI page #{page} is not a JSON array: #{body.to_s[0, 200].inspect}."
       end
 
       def process_records(records)
@@ -92,16 +187,30 @@ module Relaton
         "Published"
       end
 
+      # A page body that is not a JSON array. See #parse_page.
+      class BadPage < StandardError; end
+
+      # Seconds to wait before a bad page is fetched again after the last page.
+      DEFERRED_DELAY = 60
+
+      # Full pages read past page 1's total_count before the crawl fails.
+      MAX_EXTRA_PAGES = 10
+
       NETWORK_ERRORS = [
         Mechanize::Error, Net::OpenTimeout, Net::ReadTimeout,
         SocketError, Errno::ECONNRESET
       ].freeze
 
+      #
+      # @yield [String] the body; the block's result is returned, and a BadPage
+      #   it raises is retried like a network error
+      #
       def fetch_with_retry(url, retries: 3, delay: 2) # rubocop:disable Metrics/MethodLength
         attempt = 0
         begin
-          Mechanize.new.get(url).body
-        rescue *NETWORK_ERRORS => e
+          body = Mechanize.new.get(url).body
+          block_given? ? yield(body) : body
+        rescue *NETWORK_ERRORS, BadPage => e
           attempt += 1
           if attempt <= retries
             Util.info "Fetch failed (#{e.message}), " \