relaton 3.1.2 → 3.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4243253e77d9ff955bee5534bb5dd4dc8ace300c68b208d9ad04b0b0243468ff
-  data.tar.gz: a6737982c68329a90623a75dc407b78a8c6684373c8fe0b44e0b8a7a8bceeca9
+  metadata.gz: cf214dd5828ef52d54652cd3c01d5406527e7dd0f6be2c52447a72889e7ed3c8
+  data.tar.gz: 48c41d919e828654ae0ddfd8fc0c6dcf2dafc912b65b58b493964145eb936ca6
 SHA512:
-  metadata.gz: e3a28aa82a3b155f8222eb981cf814018539dd5b6ea4f33eb308ea034970b2670749cb778a7540767c6df7b526ffe61ed2aeb1d0f8186dfd8b121dd297d25a80
-  data.tar.gz: 4836365cff31ecc94cb9464d6c321fa8f7770d72bb40adfe963a163cd23b260a77297700596eab3c54d2eb54fbd1d76774786d4bda3f07b6765b461d4cf38f91
+  metadata.gz: 612e007cdc0a28b1eb0ac6ef9e0e563f067daa8532a25002d76835233e43b5e5051e2af133ad078a96be66d73c56c4ca58f5e3c675e51b8331cb58e3d842060c
+  data.tar.gz: 55ba69a723f91403724139457801a0fa92842208d5cc06be86fd7260a8507e85ccccbe3eb454f82553247130e1e072b1def59eee9a6c54c9b2086b8c6a29bd8d

data/lib/relaton/bib/converter/csl/from_csl.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Bib
+    module Converter
+      module Csl
+        # CSL-JSON entry -> ItemData. The downgrade direction: a CSL record
+        # is a flat citation, so fields it does not express are left out
+        # rather than invented. The mirror of Converter::Bibtex::FromBibtex.
+        class FromCsl
+          # The reverse of ENTRY_TYPES for the types the export emits; any
+          # other CSL type is kept as written.
+          TYPES = {
+            "article-journal" => "article", "chapter" => "inbook",
+            "paper-conference" => "inproceedings", "webpage" => "webresource",
+          }.freeze
+
+          # @param entry [Hash] one CSL-JSON entry
+          def initialize(entry)
+            @entry = entry
+          end
+
+          # @return [Relaton::Bib::ItemData]
+          def transform # rubocop:disable Metrics/MethodLength
+            ItemData.new(
+              id: @entry["id"],
+              docidentifier: fetch_docid,
+              type: fetch_type,
+              title: fetch_title,
+              contributor: fetch_contributor,
+              date: fetch_date,
+              abstract: fetch_abstract,
+              extent: fetch_extent,
+              edition: fetch_edition,
+              series: fetch_series,
+              source: fetch_source,
+              language: fetch_language,
+              keyword: fetch_keyword,
+              relation: fetch_relation,
+            )
+          end
+
+          private
+
+          # @return [Array<Relaton::Bib::Docidentifier>]
+          def fetch_docid
+            docid = [Docidentifier.new(content: @entry["id"].to_s, primary: true)]
+            docid << Docidentifier.new(content: @entry["ISBN"].to_s, type: "isbn") if @entry["ISBN"]
+            docid << Docidentifier.new(content: @entry["ISSN"].to_s, type: "issn") if @entry["ISSN"]
+            docid
+          end
+
+          # @return [String, nil]
+          def fetch_type
+            TYPES[@entry["type"]] || @entry["type"]
+          end
+
+          # @return [Array<Relaton::Bib::Title>]
+          def fetch_title
+            return [] unless @entry["title"]
+
+            [Title.new(type: "main", content: @entry["title"])]
+          end
+
+          # @return [Array<Relaton::Bib::Contributor>]
+          def fetch_contributor
+            contribs = []
+            fetch_person("author") { |c| contribs << c }
+            fetch_person("editor") { |c| contribs << c }
+
+            return contribs unless @entry["publisher"]
+
+            contribs << Contributor.new(
+              organization: Organization.new(name: [TypedLocalizedString.new(content: @entry["publisher"])]),
+              role: [Contributor::Role.new(type: "publisher")],
+            )
+            contribs
+          end
+
+          # CSL names are {family:, given:} or a bare {literal:}.
+          #
+          # @param role [String] contributor role
+          def fetch_person(role, &_)
+            Array(@entry[role]).each do |name|
+              yield Contributor.new(person: Person.new(name: build_name(name)), role: [Contributor::Role.new(type: role)])
+            end
+          end
+
+          # @param name [Hash] CSL name variable
+          # @return [Relaton::Bib::FullName]
+          def build_name(name)
+            return FullName.new(completename: LocalizedString.new(content: name["literal"])) if name["literal"]
+
+            fore = name["given"].to_s.split
+            FullName.new(
+              surname: LocalizedString.new(content: name["family"].to_s),
+              forename: fore.map { |fn| FullNameType::Forename.new(content: fn) },
+            )
+          end
+
+          # @return [Array<Relaton::Bib::Date>]
+          def fetch_date
+            { "issued" => "published", "accessed" => "accessed" }.filter_map do |field, type|
+              parts = @entry.dig(field, "date-parts")&.first
+              next unless parts
+
+              Bib::Date.new(type: type, at: iso_date(parts))
+            end
+          end
+
+          # @param parts [Array<Integer>] [year], [year, month] or [year, month, day]
+          # @return [String]
+          def iso_date(parts)
+            format("%04d", parts[0]).tap do |iso|
+              iso << "-%02d" % parts[1] if parts[1]
+              iso << "-%02d" % parts[2] if parts[2]
+            end
+          end
+
+          # @return [Array<Relaton::Bib::Abstract>]
+          def fetch_abstract
+            return [] unless @entry["abstract"]
+
+            [Abstract.new(content: @entry["abstract"])]
+          end
+
+          # @return [Array<Relaton::Bib::Extent>]
+          def fetch_extent
+            locs = []
+            locs << Locality.new(type: "volume", reference_from: @entry["volume"].to_s) if @entry["volume"]
+            if @entry["page"]
+              from, to = @entry["page"].to_s.split("-", 2)
+              locs << Locality.new(type: "page", reference_from: from, reference_to: to)
+            end
+            locs.any? ? [Extent.new(locality: locs)] : []
+          end
+
+          # @return [Relaton::Bib::Edition, nil]
+          def fetch_edition
+            Edition.new(content: @entry["edition"].to_s) if @entry["edition"]
+          end
+
+          # @return [Array<Relaton::Bib::Series>]
+          # container-title is the journal for article types and the book for
+          # chapters; the chapter shape maps to partOf, the journal shape to
+          # a series — the same split FromBibtex makes for journal/booktitle.
+          def fetch_series
+            return [] unless container_title && fetch_type != "inbook"
+
+            [Series.new(
+              type: "journal",
+              title: Title.new(content: container_title),
+              number: @entry["number"]&.to_s,
+            )]
+          end
+
+          # @return [Array<Relaton::Bib::Uri>]
+          def fetch_source
+            source = []
+            source << Uri.new(type: "src", content: @entry["URL"]) if @entry["URL"]
+            source << Uri.new(type: "doi", content: @entry["DOI"]) if @entry["DOI"]
+            source
+          end
+
+          # @return [Array<String>]
+          def fetch_language
+            return [] unless @entry["language"]
+
+            sub = @entry["language"].split("-").first
+            [Iso639[sub]&.alpha2 || sub]
+          end
+
+          # @return [Array<Relaton::Bib::Keyword>]
+          def fetch_keyword
+            Array(@entry["keyword"]).map do |kw|
+              Keyword.new(vocab: LocalizedString.new(content: kw))
+            end
+          end
+
+          # @return [Array<Relaton::Bib::Relation>]
+          def fetch_relation
+            return [] unless container_title && fetch_type == "inbook"
+
+            ttl = Title.new(type: "main", content: container_title)
+            [Relation.new(type: "partOf", bibitem: ItemData.new(title: [ttl]))]
+          end
+
+          def container_title
+            @entry["container-title"]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/relaton/bib/converter/csl.rb

@@ -2,6 +2,8 @@
 
 require "yeptris"
 
+require_relative "csl/from_csl"
+
 
 module Relaton
   module Bib
@@ -21,6 +23,16 @@ module Relaton
           Renderer.new(item).to_s
         end
 
+        # CSL-JSON string -> Hash{String=>ItemData}
+        def self.to_item(csl_str)
+          entries = ::Yeptris::JSON.load(csl_str)
+          entries.reduce({}) do |h, entry|
+            item = FromCsl.new(entry).transform
+            h[item.id.to_s] = item
+            h
+          end
+        end
+
         class Renderer
           def initialize(item)
             @item = item
@@ -97,7 +109,7 @@ module Relaton
           def published_year
             date = Array(@item.date).find { |d| %w[published issued].include?(d.type.to_s) } || @item.date.first
             value = date && (date.at || date.from || date.to)
-            value.to_s[/\d{4}/]
+            value.to_s[/\d{4}/] || ""
           end
 
           def source_uri

data/lib/relaton/cli/command.rb

@@ -371,7 +371,9 @@ module Relaton
       nil
     rescue Pubid::Errors::Error
       %("#{code}" is not a recognized standards identifier)
-    rescue Relaton::RequestError => e
+    rescue Relaton::RequestError, Relaton::NotInLocalStoreError => e
+      # A transport failure, or offline mode with no stored entry
+      # (relaton#536): print the reason instead of a backtrace.
       e.message
     end
 

data/lib/relaton/cli/subcommand_collection.rb

@@ -131,7 +131,11 @@ module Relaton
           File.write colfile, coll.to_yaml, encoding: "UTF-8"
         else Util.info "No matching bibliographic entry found"
         end
-      rescue Relaton::UnknownReferenceError => e
+      rescue Relaton::UnknownReferenceError, Relaton::NotInLocalStoreError,
+             Relaton::RequestError => e
+        # An unknown reference, offline mode with no stored entry
+        # (relaton#536) or a transport failure: log the reason, and do not
+        # write the collection.
         Util.warn e.message
       rescue Pubid::Errors::Error
         Util.error %("#{code}" is not a recognized standards identifier)

data/lib/relaton/core/request_error.rb

@@ -17,6 +17,24 @@ module Relaton
   # declare only when absent — both forms rescue as StandardError
   class RequestError < Error; end unless defined?(Relaton::RequestError)
 
+  # Offline mode (`Relaton::Db.configuration.offline`, RELATON_OFFLINE) and
+  # the local store has no entry for the reference (relaton#536). Not a
+  # RequestError: `Relaton::Db#net_retry` must not retry it, since no
+  # request was made.
+  class NotInLocalStoreError < Error
+    # @return [String] the reference the local store lacks
+    attr_reader :reference
+
+    # @param reference [String]
+    # @param dirs [Array<String>] the cache directories that were read
+    def initialize(reference, dirs = [])
+      @reference = reference
+      where = dirs.empty? ? "no cache directory" : dirs.join(", ")
+      super("`#{reference}` is not in the local store (#{where}); " \
+            "offline mode makes no network request")
+    end
+  end
+
   # `Relaton::Db` found no flavor for a reference: no flavor's pubid grammar
   # reads it exactly, and no flavor's prefix matches it. relaton-cli rescues
   # it, logs the message and returns no document.

data/lib/relaton/db/config.rb

@@ -11,11 +11,19 @@ module Relaton
     end
 
     class Configuration
-      attr_accessor :use_api, :api_host
+      # Values of RELATON_OFFLINE that turn offline mode on.
+      OFFLINE_ON = %w[1 true yes on].freeze
+
+      # @!attribute offline
+      #   @return [Boolean] true: every fetch is answered from the local
+      #     store only, and a miss raises Relaton::NotInLocalStoreError
+      #     (relaton#536). Defaults to RELATON_OFFLINE.
+      attr_accessor :use_api, :api_host, :offline
 
       def initialize
         @use_api = false
         @api_host = "https://api.relaton.org"
+        @offline = OFFLINE_ON.include?(ENV["RELATON_OFFLINE"].to_s.downcase)
       end
     end
 

data/lib/relaton/db/workers_pool.rb

@@ -8,7 +8,17 @@ module Relaton
         @queue = SizedQueue.new(workers * 2)
         @threads = Array.new workers do
           Thread.new do
-            while item = @queue.pop; yield(item) end
+            # A worker that dies silently wedges the pool: the item it was
+            # processing never completes, and every consumer blocked on that
+            # completion blocks forever (metanorma-standoc CI deadlock,
+            # 2026-11-13). The pool outlives any single item's failure.
+            while item = @queue.pop
+              begin
+                yield(item)
+              rescue Exception => e # rubocop:disable Lint/RescueException
+                Util.error "fetch worker item failed: #{e.class}: #{e.message}"
+              end
+            end
           end
         end
       end

data/lib/relaton/db.rb

@@ -65,21 +65,25 @@ module Relaton
     # @return [nil, RelatonBib::BibliographicItem, ...]
     # @raise [Relaton::UnknownReferenceError] no flavor recognizes the
     #   reference (see Registry#route)
+    # @raise [Relaton::NotInLocalStoreError] offline mode, and the local
+    #   store has no entry for the reference (relaton#536)
     ##
     def fetch(text, year = nil, opts = {}) # rubocop:disable Metrics/MethodLength
       reference = text.strip
-      stdclass, pubid = @registry.route(reference)
-      processor = @registry[stdclass]
-      ref = if processor.respond_to?(:urn_to_code)
-              processor.urn_to_code(reference)&.first
-            else reference
-            end
-      ref ||= reference
-      # The routed pubid is the parse of `reference`; a URN rewritten to a
-      # code is parsed again by the flavor.
-      pubid = nil unless ref == reference
-      result = combine_doc ref, year, opts, stdclass
-      result || fetch_routed(ref, year, opts, stdclass, pubid)
+      offline_answer(reference, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
+        stdclass, pubid = @registry.route(reference)
+        processor = @registry[stdclass]
+        ref = if processor.respond_to?(:urn_to_code)
+                processor.urn_to_code(reference)&.first
+              else reference
+              end
+        ref ||= reference
+        # The routed pubid is the parse of `reference`; a URN rewritten to a
+        # code is parsed again by the flavor.
+        pubid = nil unless ref == reference
+        result = combine_doc ref, year, opts, stdclass
+        result || fetch_routed(ref, year, opts, stdclass, pubid)
+      end
     end
 
     # @see Relaton::Db#fetch
@@ -124,7 +128,7 @@ module Relaton
           threads = ENV["RELATON_FETCH_PARALLEL"]&.to_i || processor.threads
           wp = WorkersPool.new(threads) do |args|
             args[3].call fetch(*args[0..2])
-          rescue Relaton::RequestError => e
+          rescue Relaton::RequestError, Relaton::NotInLocalStoreError => e
             args[3].call e
           rescue StandardError => e
             # An unparseable reference ("ISO Standards Tag Set" in a
@@ -137,6 +141,14 @@ module Relaton
               Util.error "`#{args[0]}` -- #{e.message}"
             end
             args[3].call nil
+          rescue Exception => e # rubocop:disable Lint/RescueException
+            # The completion callback is a contract: the consumer counts
+            # exactly one delivery per fetch, and a missing one blocks it
+            # forever (metanorma-standoc CI deadlock, 2026-11-13: an
+            # RSpec::Mocks::MockExpectationError -- an Exception, not a
+            # StandardError -- escaped the rescue above).
+            Util.error "`#{args[0]}` -- #{e.class}: #{e.message}"
+            args[3].call nil
           end
           @queues[stdclass] =
             { queue: SizedQueue.new(threads * 2), workers_pool: wp }
@@ -155,14 +167,19 @@ module Relaton
     # @raise [Relaton::UnknownReferenceError] no flavor is named and none
     #   recognizes the reference
     #
+    # @raise [Relaton::NotInLocalStoreError] offline mode, and the local
+    #   store has no entry for the reference (relaton#536)
+    #
     def fetch_std(code, year = nil, stdclass = nil, opts = {})
-      std = named_class(stdclass)
-      # A flavor the caller names is the one asked: no co-publisher.
-      return check_bibliocache(code, year, opts, std) if std
-
-      std, pubid = @registry.route(code)
-      fetch_routed(code, year, opts, std, pubid) ||
-        fetch_equivalent(code, year, opts)
+      offline_answer(code, opts) do |opts| # rubocop:disable Lint/ShadowingOuterLocalVariable
+        std = named_class(stdclass)
+        # A flavor the caller names is the one asked: no co-publisher.
+        next check_bibliocache(code, year, opts, std) if std
+
+        std, pubid = @registry.route(code)
+        fetch_routed(code, year, opts, std, pubid) ||
+          fetch_equivalent(code, year, opts)
+      end
     end
 
     # Cross-SDO equivalence as data: on a total miss, walk the equivalence
@@ -174,6 +191,9 @@ module Relaton
     #
     # @return [Relaton::Bib::ItemData, nil]
     def fetch_equivalent(code, year, opts)
+      # `relaton/db` alone does not declare the `Relaton::Equivalence`
+      # autoload (`relaton.rb` does), so a fetch_std miss NameErrored there.
+      require_relative "equivalence"
       Equivalence.related_for(code).each do |candidate|
         Util.info "Not found; trying equivalent `#{candidate}`", key: code
         std, pubid = @registry.route(candidate)
@@ -221,6 +241,24 @@ module Relaton
 
     private
 
+    #
+    # Offline mode (Configuration#offline, relaton#536) answers every fetch
+    # from the local store only: the block runs with `fetch_db` set, so no
+    # path reaches a flavor's `get`, and a miss of the whole resolution
+    # (routed flavor, co-publishers, equivalents) raises. An explicit
+    # `fetch_db` keeps its nil answer.
+    #
+    # @yieldparam opts [Hash] the options to fetch with
+    # @raise [Relaton::NotInLocalStoreError]
+    #
+    def offline_answer(reference, opts)
+      return yield(opts) if opts[:fetch_db] || !Db.configuration.offline
+
+      yield(opts.merge(fetch_db: true)) or
+        raise NotInLocalStoreError.new(reference.to_s,
+                                       [@local_db, @db].compact.map(&:dir))
+    end
+
     # True when the error is a flavor's grammar refusing to read the
     # reference. Both are raised by the flavors' pubid/parslet parses; a
     # reference they cannot read is not-found, not a failure (relaton#412).
@@ -489,30 +527,48 @@ module Relaton
         return check_date_range(searchcode, id, year, opts, stdclass, query)
       end
 
-      @semaphore.synchronize { db.expire id, year }
-      if altdb
-        return bib_retval(altdb.read(id), stdclass) if opts[:fetch_db]
+      # The cache-only path reads, and never expires: a stale entry is
+      # still the local store's answer (relaton#536).
+      # A document in either cache answers over a not_found in the other.
+      if opts[:fetch_db]
+        entries = [db, altdb].compact.map { |c| c.read(id) }
+        return bib_retval(entries.find { |e| e.is_a?(String) }, stdclass)
+      end
 
-        @semaphore.synchronize do
-          db.clone_entry id, altdb if altdb.valid_entry? id, year
-        end
-        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
-                      query: query)
-        @semaphore.synchronize do
-          altdb.clone_entry(id, db) if !altdb.valid_entry?(id, year)
-        end
-      else
-        return bib_retval(db.read(id), stdclass) if opts[:fetch_db]
+      @semaphore.synchronize { replace_stale(db, altdb, id, year) } if altdb
+      # The entry new_bib_entry answers, not a second read of `db`: a stale
+      # document served from `altdb` has no row in `db`.
+      entry = new_bib_entry(searchcode, year, opts, stdclass,
+                            db: db, id: id, query: query, altdb: altdb)
+      @semaphore.synchronize { replace_stale(altdb, db, id, year) } if altdb
+      bib_retval(entry, stdclass)
+    end
 
-        new_bib_entry(searchcode, year, opts, stdclass, db: db, id: id,
-                      query: query)
-      end
-      bib_retval(db.read(id), stdclass)
+    # Replace the stale (or missing) row of `id` in `cache` with the valid
+    # row `other` holds. A valid row of `cache` is kept (local over global).
+    def replace_stale(cache, other, id, year)
+      return if cache.valid_entry?(id, year)
+      return unless other.valid_entry?(id, year)
+
+      cache.expire id, year
+      cache.clone_entry id, other
     end
 
+    #
+    # A stale entry (Cache#valid_entry? is false) is refetched, but it is
+    # not deleted first: Db#fetch_entry serves it when the refetch fails
+    # with a transport error, and replaces it when the refetch answers.
+    # With no row of its own, the stale document of the other cache
+    # (`altdb:`) is the fallback. `no_cache` asks for a fresh answer, so it
+    # has no fallback.
+    #
     def new_bib_entry(code, year, opts, stdclass, **args)
-      entry = @semaphore.synchronize { args[:db]&.read(args[:id]) }
-      if !entry || opts[:no_cache]
+      entry, valid = @semaphore.synchronize do
+        found = args[:db]&.read(args[:id])
+        [found, found && args[:db].valid_entry?(args[:id], year)]
+      end
+      if !entry || opts[:no_cache] || !valid
+        args = with_stale(args, entry) unless opts[:no_cache]
         return fetch_entry(code, year, opts, stdclass, **args)
       end
 
@@ -524,18 +580,47 @@ module Relaton
       entry
     end
 
-    def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize
+    # @return [Hash] args, with the `stale:` entry and the `stale_db:` cache
+    #   that holds it, when there is one
+    def with_stale(args, entry)
+      stale_db = args[:db]
+      unless entry
+        stale_db = args[:altdb]
+        entry = @semaphore.synchronize { stale_db&.read(args[:id]) }
+        return args unless entry.is_a?(String)
+      end
+      args.merge(stale: entry, stale_db: stale_db)
+    end
+
+    #
+    # @option args [String, Relaton::Db::NotFound, nil] :stale the stale
+    #   entry of the key: a transport failure answers a stale document with
+    #   a warning (relaton#536), and an answer replaces it
+    # @option args [Relaton::Db::Cache] :stale_db the cache that holds it
+    #
+    def fetch_entry(code, year, opts, stdclass, **args) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
       processor = @registry[stdclass]
-      bib = net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
-                      args[:query])
+      bib = begin
+        net_retry(code, year, opts, processor, opts.fetch(:retries, 1),
+                  args[:query])
+      rescue Relaton::RequestError => e
+        raise unless args[:stale].is_a?(String)
+
+        Util.warn "#{e.message}; using the cached entry fetched " \
+                  "#{args[:stale_db].fetched(args[:id])}", key: code
+        return args[:stale]
+      end
       entry = bib_entry bib
       return entry if args[:db].nil?
 
       # `no_cache` refreshes a cached entry, but a failed fetch does not
-      # replace a cached document with a not-found entry.
+      # replace a valid cached document with a not-found entry. A stale
+      # entry is replaced by any answer; its row goes first, so a dated
+      # query's subset row does not outlive the refetch.
       refresh = opts[:no_cache] && bib.respond_to?(:to_xml)
       @semaphore.synchronize do
-        if refresh || !args[:db].read(args[:id])
+        args[:db].expire args[:id], year if args[:stale]
+        if args[:stale] || refresh || !args[:db].read(args[:id])
           save_bib args[:db], args[:id], bib, entry, stdclass
         end
       end

data/lib/relaton/etsi/data_fetcher.rb

@@ -166,17 +166,27 @@ module Relaton
 
       def normalize(record)
         {
-          "ETSI deliverable" => record["ETSI_DELIVERABLE"],
-          "title" => record["TITLE"],
+          "ETSI deliverable" => sanitize(record["ETSI_DELIVERABLE"]),
+          "title" => sanitize(record["TITLE"]),
           "Details link" => "https://webapp.etsi.org/workprogram/Report_WorkItem.asp?WKI_ID=#{record['wki_id']}",
           "PDF link" => "https://www.etsi.org/deliver/#{record['EDSpathname']}#{record['EDSPDFfilename']}",
           "Status" => derive_status(record),
-          "Keywords" => record["Keywords"].to_s,
-          "Technical body" => record["TB"],
-          "Scope" => record["Scope"],
+          "Keywords" => sanitize(record["Keywords"]).to_s,
+          "Technical body" => sanitize(record["TB"]),
+          "Scope" => sanitize(record["Scope"]),
         }
       end
 
+      # The upstream ETSI metadata carries stray control characters — one
+      # 1998-era record ships a literal NUL inside "sequences", which the YAML
+      # dump surfaces as a raw \0 escape and which silently truncates the
+      # api.relaton.org D1 import stream (relaton-api-js#155,
+      # relaton-data-etsi#7). Non-structural C0 controls are dropped; \t, \n
+      # and \r stay — the Scope column is hard-wrapped upstream text.
+      def sanitize(value)
+        value&.gsub(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/, "")
+      end
+
       def derive_status(record)
         return "Withdrawn" if record["ACTION_TYPE"] == "WD"
 

data/lib/relaton/jis/README.adoc

@@ -4,6 +4,8 @@ RelatonJis is a Ruby gem that implements the https://github.com/metanorma/metano
 
 You can use it to retrieve metadata of JIS Standards from https://webdesk.jsa.or.jp, and access such metadata through the `Relaton::Jis::ItemData` object.
 
+The flavor also retrieves WES standards of the Japan Welding Engineering Society (JWES), which webdesk.jsa.or.jp sells next to JIS: `Relaton::Jis::Bibliography.get "WES 7700-1:2019"`. A WES record has the docidentifier type `WES` and JWES as its author and publisher. Its doctype is the JIS value of its form (`japanese-industrial-standard`, `technical-report`, `technical-specification` or `amendment`), because the JIS grammar has no WES value.
+
 == Usage
 
 === Search for standards using keywords

data/lib/relaton/jis/data_fetcher.rb

@@ -9,11 +9,20 @@ module Relaton
     class DataFetcher < Core::DataFetcher
       URL = "https://webdesk.jsa.or.jp/books/"
 
+      # The Webdesk lists a crawl reads, in this order: the JIS search, then
+      # the JWES group list (WES standards, relaton-jis#18). Webdesk keeps the
+      # open list in the session, and the list page and its paging
+      # (`W11M0070`) read that list. So the lists are read one after the
+      # other, and a session refresh opens the current list again.
+      LISTS = %w[JIS JWES].freeze
+
       def initialize(output, format)
         super
         @queue = SizedQueue.new 10
         @threads = create_thread_pool 5
         @mutex = Mutex.new
+        # URLs of the documents that did not fetch; see #prune_unwritten.
+        @failed_docs = []
       end
 
       def log_error(msg)
@@ -72,30 +81,63 @@ module Relaton
           else
             Util.warn "URL: #{url}"
             Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
+            @mutex.synchronize { @failed_docs << url }
           end
         else
           save_doc bib, url
         end
       end
 
+      # relaton-data-jis crawls with `prune: true`: it keeps data/, and after
+      # #fetch, #prune_unwritten deletes each file the run did not write. So a
+      # list that does not open, or stops part way (#get_next_page), raises:
+      # the crawl job fails before the prune and publishes nothing.
       def fetch(_source = nil)
-        return unless initial_post
+        failed = begin
+          LISTS.find { |list| !fetch_list(list) }
+        ensure
+          end_threads_and_wait
+        end
+        raise Error, "Webdesk list #{failed} did not open" if failed
 
-        resp = agent.get "#{URL}W11M0070/index"
-        parse_page resp
         index.save
         report_errors
       end
 
+      # Queue every document of one list. False if the list does not open.
+      def fetch_list(list)
+        @list = list
+        @initial_time = nil
+        return false unless initial_post
+
+        parse_page agent.get("#{URL}W11M0070/index")
+        true
+      end
+
+      # Open the current list in the session, unless it was opened less than
+      # 10 minutes ago.
       def initial_post
         return true if @initial_time && Time.now - @initial_time < 600
 
+        status = @list == "JWES" ? open_group_list(@list) : open_jis_search
+        @initial_time = Time.now
+        status || Util.warn("No results found for #{@list}")
+      end
+
+      def open_jis_search
         body = { record: 0, dantai: "JIS", searchtype2: 1,
                  status_1: 1, status_2: 2 } # rubocop:disable Naming/VariableNumber
         resp = agent.post "#{URL}W11M0270/index", body
-        disp = Relaton::Json.parse_external resp.body
-        @initial_time = Time.now
-        disp["status"] || Util.warn("No results found for JIS")
+        Relaton::Json.parse_external(resp.body)["status"]
+      end
+
+      # A group list from the "domestic group publications" page. The link
+      # redirects to the list page, which states the count of documents.
+      def open_group_list(dantai)
+        page = agent.get "#{URL}W11S0260/getSearchAllListByLink" \
+                         "?dantai_cd=#{dantai}&display=KOKUNAI"
+        script = page.at_xpath('//script[contains(.,"var count =")]')
+        script&.text.to_s[/var count = (\d+);/, 1].to_i.positive?
       end
 
       def agent
@@ -111,7 +153,6 @@ module Relaton
 
           resp = get_next_page(offset)
         end
-        end_threads_and_wait
       end
 
       def parse_offset(resp) # rubocop:disable Metrics/AbcSize
@@ -135,25 +176,44 @@ module Relaton
         @count.to_i
       end
 
+      # The next page of the current list. Raises when the page does not
+      # come, because a list that stops part way leaves its unread documents
+      # unwritten, and the prune would delete them.
       def get_next_page(offset) # rubocop:disable Metrics/MethodLength
         attempts = 0
         begin
-          if initial_post
-            url = "#{URL}W11M0070/getAddList"
-            agent.post url, search_type: "JIS", offset: offset
+          unless initial_post
+            raise Error, "Webdesk list #{@list} did not open again " \
+                         "at offset #{offset}"
           end
+
+          agent.post "#{URL}W11M0070/getAddList",
+                     search_type: "JIS", offset: offset
+        rescue Error
+          raise
         rescue StandardError => e
           attempts += 1
           if attempts < 5
             sleep 2
             retry
-          else
-            Util.warn "#{e.message}\n#{e.backtrace[0..6].join("\n")}"
           end
+          raise Error, "Webdesk list #{@list} stopped at offset " \
+                       "#{offset}: #{e.message}"
         end
       end
 
-      def save_doc(bib, url) # rubocop:disable Metrics/MethodLength
+      # Prune only after a complete crawl: a document that did not fetch was
+      # not written, and the prune would delete its file.
+      def prune_unwritten(min_ratio: 0.9)
+        unless @failed_docs.empty?
+          warn "prune skipped: #{@failed_docs.size} document(s) did not fetch"
+          return []
+        end
+
+        super
+      end
+
+      def save_doc(bib, url) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
         return unless bib
 
         id = bib.docidentifier.find(&:primary).content
@@ -175,6 +235,7 @@ module Relaton
           Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
           @files << file
           File.write file, serialize(bib), encoding: "UTF-8"
+          record_written file
           pid = pubid id
           index.add_or_update pid, file if pid
         end

data/lib/relaton/jis/hit_collection.rb

@@ -83,7 +83,8 @@ module Relaton
 
       def create_relation(hit)
         id = hit.pubid.to_s
-        docid = Docidentifier.new content: id, type: "JIS", primary: true
+        type = id.start_with?("WES ") ? "WES" : "JIS"
+        docid = Docidentifier.new content: id, type: type, primary: true
         bibitem = ItemData.new(
           formattedref: Bib::Formattedref.new(content: id),
           docidentifier: [docid],

data/lib/relaton/jis/processor.rb

@@ -11,7 +11,7 @@ module Relaton
         @short = :relaton_jis
         @prefix = "JIS"
         @pubid_identifier = :Jis # Db cache key
-        @defaultprefix = %r{^(JIS|TR)\s}
+        @defaultprefix = %r{^(JIS|TR|WES)\s} # WES: relaton-jis#18
         @idtype = "JIS"
         @datasets = %w[jis-webdesk]
       end

data/lib/relaton/jis/scraper.rb

@@ -15,6 +15,12 @@ module Relaton
       DATETYPES = { "発行年月日" => "issued", "確認年月日" => "confirmed" }.freeze
       STATUSES = { "有効" => "valid", "廃止" => "withdrawn" }.freeze
 
+      # WES (Japan Welding Engineering Society) standards are sold on the same
+      # site and kept in the JIS flavor (relaton-jis#18). Their pages name no
+      # drafting body, so JWES is the author and publisher.
+      JWES_NAMES = { "ja" => "一般社団法人日本溶接協会",
+                     "en" => "Japan Welding Engineering Society" }.freeze
+
       def initialize(url, errors = {})
         @url = url
         @agent = Mechanize.new
@@ -70,13 +76,17 @@ module Relaton
         return [] if docid.nil? || docid.empty?
 
         [Docidentifier.new(
-          content: docid, type: "JIS", primary: true,
+          content: docid, type: id_type, primary: true,
         )]
       end
 
+      # The series letter and the number: `JIS X 0208:1997` -> `X0208`,
+      # `TR Z 0010:2008` -> `Z0010`. A WES id has no series letter:
+      # `WES TR 2032:2024` -> `2032`.
       def fetch_docnumber
-        docid = document_id
-        match = docid&.match(/^\w+\s(\w)\s?(\d+)/)
+        match = document_id&.match(
+          /^(?:JIS\s)?(?:WES\s)?(?:T[RS]\s)?(?:([A-Z])\s?)?(\d+)/,
+        )
         @errors[:docnumber] &&= match.nil?
         return unless match
 
@@ -87,6 +97,15 @@ module Relaton
         @document_id ||= @doc.at_xpath("./h2/text()[1]")&.text&.strip
       end
 
+      def wes?
+        document_id.to_s.start_with? "WES "
+      end
+
+      # The docidentifier and structuredidentifier type.
+      def id_type
+        wes? ? "WES" : "JIS"
+      end
+
       def fetch_date
         result = DATETYPES.each_with_object([]) do |(key, type), a|
           node = @doc.at_xpath("./div/div/div/p/text()[contains(.,'#{key}')]")
@@ -141,8 +160,13 @@ module Relaton
         Bib::Status.new(stage: stage)
       end
 
+      # The JIS grammar requires a doctype and has no WES value, so a WES
+      # record takes the JIS value of its form (`WES TR`/`WES TS` match the
+      # TR/TS rules).
       def fetch_doctype # rubocop:disable Metrics/CyclomaticComplexity
         type = case document_id
+               when %r{^WES\s\d[^/]*/AMENDMENT} then "amendment"
+               when /^WES\s\d/ then "japanese-industrial-standard"
                when /JIS\s[A-Z]\s[\w-]+:\d{4}\/AMENDMENT/ then "amendment"
                when /JIS\s[A-Z]\s[\w-]+/ then "japanese-industrial-standard"
                when /TR[\s\/][\w-]+/ then "technical-report"
@@ -163,6 +187,8 @@ module Relaton
       end
 
       def fetch_contributor
+        return jwes_contributors if wes?
+
         authorizer = create_contrib(
           "一般財団法人　日本規格協会", "authorizer"
         )
@@ -175,6 +201,20 @@ module Relaton
         result
       end
 
+      def jwes_contributors
+        tls = Bib::TypedLocalizedString
+        scripts = { "ja" => "Jpan", "en" => "Latn" }
+        %w[author publisher].map do |role|
+          name = JWES_NAMES.map do |lang, content|
+            tls.new content: content, language: lang, script: scripts[lang]
+          end
+          org = Bib::Organization.new name: name
+          Bib::Contributor.new(
+            organization: org, role: [Bib::Contributor::Role.new(type: role)],
+          )
+        end
+      end
+
       def create_contrib(name, role)
         org = Bib::Organization.new name: create_orgname(name)
         role_obj = Bib::Contributor::Role.new(type: role)
@@ -195,7 +235,8 @@ module Relaton
 
       def fetch_editorialgroup_contributor # rubocop:disable Metrics/MethodLength
         node = @doc.at_xpath("./table/tr[th[.='原案作成団体']]/td")
-        @errors[:editorialgroup] &&= node.nil?
+        # A WES page names no drafting body (see JWES_NAMES).
+        @errors[:editorialgroup] &&= node.nil? unless wes?
         return unless node
 
         subdivision = Bib::Subdivision.new(
@@ -215,7 +256,7 @@ module Relaton
       def fetch_structuredidentifier
         StructuredIdentifier.new(
           project_number: Iso::ProjectNumber.new(content: fetch_docnumber),
-          type: "JIS",
+          type: id_type,
         )
       end
 

data/lib/relaton/plateau/data_fetcher.rb

@@ -12,6 +12,10 @@ module Relaton
     class DataFetcher < Core::DataFetcher
       HANDBOOKS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/handbooks.json".freeze
       TECHNICAL_REPORTS_URL = "https://www.mlit.go.jp/plateau/_next/data/1.3.0/libraries/technical-reports.json".freeze
+      # The Japanese volume label of a Handbook id gets an ASCII file-name
+      # suffix: #file_name strips non-word characters, so 本編 and 本編＋付録
+      # of one edition would otherwise share a file.
+      VOLUME_SUFFIX = { "本編＋付録" => "-with-appendix", "本編" => "-main" }.freeze
 
       def index
         @index ||= Relaton::Index.find_or_create(
@@ -125,6 +129,9 @@ module Relaton
           Util.warn "File #{file} already exists, skipping.", key: id
         else
           File.write(file, serialize(item))
+          # Recorded for #prune_unwritten: the data repo crawls into a kept
+          # data/, and an unrecorded write makes every prune skip.
+          record_written file
           @files << file
           pid = pubid id
           if pid
@@ -152,12 +159,14 @@ module Relaton
       end
 
       def file_name(id)
-        name = id.gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
+        volume = id[/ (本編(?:＋付録)?)\z/, 1]
+        name = id.delete_suffix(" #{volume}").gsub(/\s+/, "-").gsub(/[^\w-]+/, "").downcase
         if id.match?(/民間活用編/)
           name += "-private"
         elsif id.match?(/公共活用編/)
           name += "-public"
         end
+        name += VOLUME_SUFFIX[volume] if volume
         File.join(@output, "#{name}.#{@ext}")
       end
 

data/lib/relaton/plateau/handbook_parser.rb

@@ -23,6 +23,13 @@ module Relaton
         @edition_label ||= @version["title"].to_s[/第[\d.]+版/].to_s
       end
 
+      # Volume label of the version title, as Pubid::Plateau renders it:
+      # "本編" (main volume) or "本編＋付録" (main volume and appendix). Handbook
+      # #00 edition 5.0 is published as both. ASCII "+" becomes "＋".
+      def volume_label
+        @version["title"].to_s[/第[\d.]+版[ 　]+(本編(?:[＋+]付録)?)/, 1]&.tr("+", "＋")
+      end
+
       def slug_number
         @slug_number ||= mapped_docnumber&.[](/#(\S+)/, 1) ||
           @entry["slug"]&.to_s&.split("_")&.first
@@ -31,7 +38,7 @@ module Relaton
       def parse_docnumber
         @errors[:hb_docnumber] &&= @entry["slug"].nil? || @entry["slug"].to_s.empty?
         mapped_docnumber ||
-          ["Handbook ##{slug_number}", edition_label].reject { |s| s.to_s.empty? }.join(" ")
+          ["Handbook ##{slug_number}", edition_label, volume_label].reject { |s| s.to_s.empty? }.join(" ")
       end
 
       # Part label of a split edition, e.g. "（公共活用編）". Editions 1-3 of

data/lib/relaton/plateau/hit_collection.rb

@@ -9,16 +9,24 @@ module Relaton
       # legacy Latin form (`1.0`), stripped in #to_all_editions to derive the
       # edition-less family id from a relaton-bib docidentifier *string* (the
       # fetched document is not a pubid). Matching itself is done on pubid objects.
-      EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)$/
+      # A Handbook volume label (`本編`, `本編＋付録`) follows the edition, and
+      # goes with it.
+      EDITION_SUFFIX = / (?:第[\d.]+版|\d+\.\d+)(?: 本編(?:＋付録)?)?$/
 
       # An edition-less reference selects every edition of the document with
       # pubid's subset match `pubid_ref === row`: the omitted edition matches
       # any value, and pubid declares `annex` strict for PLATEAU, so
       # `PLATEAU Handbook #03` does not reach `#03-1`. A reference with an
-      # edition selects that one row with `==`.
+      # edition selects that one row with `==`. pubid also declares `volume`
+      # strict, so the edition-less match compares each row without its
+      # volume (`本編`, `本編＋付録`): every edition includes every volume.
       def find
-        @array = index.search(pubid_ref, exact: !all_editions?)
-          .map { |row| Hit.new(row, self) }
+        rows = if all_editions?
+                 index.search(pubid_ref) { |row| pubid_ref === without_volume(row[:id]) }
+               else
+                 index.search(pubid_ref, exact: true)
+               end
+        @array = rows.map { |row| Hit.new(row, self) }
         self
       end
 
@@ -49,6 +57,13 @@ module Relaton
         @pubid_ref ||= ref.is_a?(String) ? ::Pubid::Plateau.parse(ref) : ref
       end
 
+      # The row's identifier as an edition-less reference sees it. `#exclude`
+      # returns a copy, so the cached index row is untouched. Only a Handbook
+      # has a volume.
+      def without_volume(id)
+        id.respond_to?(:volume) && id.volume ? id.exclude(:volume) : id
+      end
+
       # A reference with no edition (`PLATEAU Handbook #00`, or any Technical
       # Report — TRs carry no edition) asks for all editions of the document.
       def all_editions?

data/lib/relaton/version.rb

@@ -1,3 +1,3 @@
 module Relaton
-  VERSION = "3.1.2".freeze
+  VERSION = "3.1.3".freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: relaton
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.1.3
 platform: ruby
 authors:
 - Ribose Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-11-11 00:00:00.000000000 Z
+date: 2026-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -520,14 +520,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.9
+        version: 2.0.22
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.9
+        version: 2.0.22
 - !ruby/object:Gem::Dependency
   name: yeptris
   requirement: !ruby/object:Gem::Requirement
@@ -654,6 +654,7 @@ files:
 - lib/relaton/bib/converter/bibxml/to_rfcxml_referencegroup_v3.rb
 - lib/relaton/bib/converter/bibxml/to_rfcxml_v3.rb
 - lib/relaton/bib/converter/csl.rb
+- lib/relaton/bib/converter/csl/from_csl.rb
 - lib/relaton/bib/converter/ris.rb
 - lib/relaton/bib/converter/titles.rb
 - lib/relaton/bib/hash_parser_v1.rb