relaton 3.0.6 → 3.0.8

data/lib/relaton/nist/hit_collection.rb

@@ -12,6 +12,12 @@ module Relaton
 
       GHNISTDATA = "https://raw.githubusercontent.com/relaton/relaton-data-nist/v2/"
 
+      # Errors of the local CSRC cache (read-only or full disk, a bad zip).
+      LOCAL_DATA_ERRORS = [
+        Errno::EROFS, Errno::EACCES, Errno::EPERM, Errno::ENOSPC,
+        Errno::ENOENT, Zip::Error
+      ].freeze
+
       attr_reader :reference
       attr_accessor :array
 
@@ -47,7 +53,7 @@ module Relaton
       # @return [Relaton::Nist::HitCollection] hits collection
       #
       def search
-        @array = from_json
+        @array = from_json_or_empty
         @array = from_ga unless @array.any?
         sort_hits!
       end
@@ -400,6 +406,24 @@ module Relaton
         select_data.map { |h| Hit.new(hit_data(h), self) }
       end
 
+      #
+      # The CSRC hits, or none when the local CSRC data cannot be used: a
+      # file-system error must not stop the search, because the
+      # relaton-data-nist index can still answer (relaton-nist#57). Network
+      # errors are not rescued: they still reach Bibliography.search, which
+      # raises Relaton::RequestError, so a CSRC outage is not cached as
+      # "not found".
+      #
+      # @return [Array<Relaton::Nist::Hit>] hits
+      #
+      def from_json_or_empty
+        from_json
+      rescue *LOCAL_DATA_ERRORS => e
+        Util.warn "CSRC pubs-export data is not available (#{e.message}); " \
+                  "searching the Relaton repository only", key: @reference
+        []
+      end
+
       def pubs_export_id(json)
         self.class.pubs_export_id json
       end

data/lib/relaton/nist/mods_parser.rb

@@ -13,7 +13,7 @@ module Relaton
       }.freeze
 
       ATTRS = %i[type docidentifier title source abstract date contributor
-                  relation place series].freeze
+                  relation place series edition].freeze
 
       def initialize(doc, series, errors = {})
         @doc = doc
@@ -51,17 +51,41 @@ module Relaton
       def get_id_from_str(str)
         return if str.nil? || str.empty?
 
-        # DOIs arrive dotted ("NIST.HB.135e2022-upd1"); that MR-style form
-        # parses through a path that skips UpdateCodes date lookup and drops
-        # letter suffixes. Space the leading "PUB.SERIES." prefix so it parses
-        # like the canonical short form (only the separator dots are touched,
-        # not any inside the number). Then force :human rendering.
-        spaced = str.sub(/\A(NIST|NBS)\.([A-Z][A-Za-z]*)\./, '\1 \2 ')
-        ::Pubid::Nist::Identifier.parse(spaced).to_s(format: :human)
+        parse_pubid(str).to_s(format: :human)
       rescue ::Pubid::Errors::ParseError
         str.gsub(".", " ").sub(/^[\D]+/, &:upcase)
       end
 
+      # DOIs arrive dotted ("NIST.HB.135e2022-upd1"); that MR-style form
+      # parses through a path that skips UpdateCodes date lookup and drops
+      # letter suffixes. Space the leading "PUB.SERIES." prefix so it parses
+      # like the canonical short form (only the separator dots are touched,
+      # not any inside the number).
+      def parse_pubid(str)
+        spaced = str.sub(/\A(NIST|NBS)\.([A-Z][A-Za-z]*)\./, '\1 \2 ')
+        ::Pubid::Nist::Identifier.parse(spaced)
+      end
+
+      # The edition number a DOI states ("NBS.CIRC.111e2" -> 2), relaton-nist#56.
+      # The MODS carries no edition number of its own (its <edition> is prose:
+      # "New version", "Erratum"), so the DOI is the only source. Only a
+      # numbered "e<N>" edition counts: an "e<year>" edition is a date, and an
+      # "r<N>" revision is not an edition. An edition 0 counts as 1 (the issue's
+      # second rule; no live DOI has one).
+      # @return [Bib::Edition, nil]
+      def parse_edition
+        doi = parse_doi
+        return if doi.nil? || doi.empty?
+
+        edition = parse_pubid(doi).edition
+        return unless edition&.type == "e" && edition.id.to_s.match?(/\A\d{1,3}\z/)
+
+        number = edition.id.to_i.zero? ? "1" : edition.id
+        Bib::Edition.new(number: number, content: number)
+      rescue ::Pubid::Errors::Error, ArgumentError
+        nil
+      end
+
       # @return [String]
       def replace_wrong_doi(id)
         case id

data/lib/relaton/nist/pubs_export.rb

@@ -8,6 +8,8 @@ module Relaton
 
       DOMAIN = "https://csrc.nist.gov".freeze
       PUBS_EXPORT = URI.join(DOMAIN, "/CSRC/media/feeds/metanorma/pubs-export")
+      # The default cache location. The live location is #datafile, which
+      # follows Relaton::Index.config.storage_dir (default Dir.home).
       DATAFILEDIR = File.expand_path ".relaton/nist", Dir.home
       DATAFILE = File.expand_path "pubs-export.zip", DATAFILEDIR
 
@@ -23,8 +25,8 @@ module Relaton
       def data
         @mutex.synchronize do
           @data ||= begin
-            ctime = File.ctime DATAFILE if File.exist? DATAFILE
-            if !ctime || ctime.to_date < ::Date.today || File.size(DATAFILE).zero?
+            ctime = File.ctime datafile if File.exist? datafile
+            if !ctime || ctime.to_date < ::Date.today || File.size(datafile).zero?
               fetch_data(ctime)
             end
             unzip
@@ -32,22 +34,64 @@ module Relaton
         end
       end
 
+      #
+      # The cache file of the pubs-export zip. It lives under the same root as
+      # the relaton-index cache, so one setting moves both (for example to a
+      # writable directory on a read-only host).
+      #
+      # @return [String]
+      #
+      def datafile
+        dir = Dir.home
+        dir = Relaton::Index.config.storage_dir if defined?(Relaton::Index)
+        File.expand_path "pubs-export.zip", File.join(dir, ".relaton", "nist")
+      end
+
       private
 
       #
-      # Fetch data form server and save it to file
+      # Fetch data form server and save it to file. When the cache file cannot
+      # be written (a read-only file system, such as AWS Lambda —
+      # relaton-nist#57), keep the downloaded zip in memory for this process.
       #
       # @param ctime [Time, nil] file creation time
       #
       def fetch_data(ctime)
-        if !ctime || ctime < last_modified
-          @data = nil
-          uri_open = URI.method(:open) || Kernel.method(:open)
-          FileUtils.mkdir_p DATAFILEDIR
-          IO.copy_stream(uri_open.call("#{PUBS_EXPORT}.zip"), DATAFILE)
+        return unless !ctime || ctime < last_modified
+
+        @data = nil
+        zip = download
+        begin
+          write_datafile zip
+          @buffer = nil
+        rescue SystemCallError, IOError => e
+          Util.warn "Cannot write `#{datafile}` (#{e.message}); " \
+                    "the CSRC pubs-export data stays in memory"
+          @buffer = zip
         end
       end
 
+      #
+      # Write the zip to a temp file and rename it into place, so a failed
+      # write (a full disk) never leaves a truncated cache that later
+      # processes would read as fresh.
+      #
+      # @param zip [String] zip file content
+      #
+      def write_datafile(zip)
+        FileUtils.mkdir_p File.dirname(datafile)
+        tmp = "#{datafile}.#{Process.pid}.tmp"
+        File.binwrite tmp, zip
+        File.rename tmp, datafile
+      ensure
+        FileUtils.rm_f tmp if tmp && File.exist?(tmp)
+      end
+
+      def download
+        uri_open = URI.method(:open) || Kernel.method(:open)
+        uri_open.call("#{PUBS_EXPORT}.zip", &:read)
+      end
+
       def last_modified
         uri = URI("#{PUBS_EXPORT}.meta")
         Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
@@ -56,15 +100,29 @@ module Relaton
       end
 
       #
-      # upack zip file
+      # upack zip file (the in-memory copy when the cache was not writable)
       #
       # @return [Array<Hash>] json data
       #
       def unzip
-        Zip::File.open(DATAFILE) do |zf|
-          Relaton::Json.parse_external zf.first.get_input_stream.read
+        if @buffer
+          # open_buffer returns the buffer, not the block's value
+          rows = nil
+          Zip::File.open_buffer(StringIO.new(@buffer)) do |zf|
+            rows = read_entry zf
+          end
+          @buffer = nil
+          rows
+        else
+          Zip::File.open(datafile) { |zf| read_entry zf }
         end
       end
+
+      # The block form closes the entry stream. Without it the cache file
+      # stays open, and Windows then refuses to replace or delete it.
+      def read_entry(zip_file)
+        Relaton::Json.parse_external zip_file.first.get_input_stream(&:read)
+      end
     end
   end
 end

data/lib/relaton/ogc/CLAUDE.md

@@ -203,7 +203,7 @@ Models declare attributes and XML mappings in the same class. `Item` is the seri
 - **RSpec** with `--format documentation`
 - **VCR** + **WebMock** for HTTP interaction recording (`spec/vcr_cassettes/`)
 - **Jing** for RelaxNG schema validation against `grammars/relaton-ogc-compile.rng`
-- **equivalent-xml** for XML comparison in tests
+- **canon** for XML comparison in tests
 - Fixtures live in `spec/fixtures/` (YAML, XML)
 - Integration tests in `spec/relaton/ogc/integration_spec.rb` exercise the full search → fetch → serialize pipeline
 

data/lib/relaton/omg/CLAUDE.md

@@ -124,7 +124,7 @@ Items can be serialized to/from YAML and XML. Tests verify round-trip fidelity f
 ### Test patterns
 
 - RSpec with `expect` syntax (monkey patching disabled)
-- `equivalent-xml` for XML comparison
+- `canon` for XML comparison
 - `ruby-jing` for RELAX NG schema validation against `grammars/relaton-omg-compile.rng`
 - VCR cassettes in `spec/vcr_cassettes/` for HTTP interaction recording
 - Fixtures in `spec/fixtures/` (YAML and XML reference files)

data/lib/relaton/plateau/CLAUDE.md

@@ -114,7 +114,7 @@ RNG grammar files in `grammars/` define the XML schema. Tests validate fixtures
 - **Index fixture:** `spec/fixtures/index-v2.zip` (pubid-backed, `_type: pubid:plateau:*` rows) is pre-loaded into the `Relaton::Index` pool in `before(:suite)` via a `Relaton::Index::Type` built with `::Pubid::Plateau::Identifier` (configured in `spec/support/webmock.rb`, mirroring NIST/CIE). Run `rake spec:update_index` to refresh from relaton-data-plateau.
 - RSpec with `expect` syntax (no monkey-patching)
 - VCR cassettes record HTTP interactions (in `spec/fixtures/vcr_cassettes/`)
-- `equivalent-xml` for XML comparison assertions
+- `canon` for XML comparison assertions
 - `jing` matcher for RNG schema validation
 - SimpleCov for code coverage
 

data/lib/relaton/render/general.rb

@@ -0,0 +1,319 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Render
+    # Compatibility entry point (relaton-render#90).
+    #
+    # The v2 engine replaced the liquid/config/i18n machinery with the
+    # template evaluator over CitationStyle instances
+    # (Relaton::Render::Iso690). isodoc main's render-isodoc subclasses
+    # this constant, so it must exist for all-git-main stacks to boot.
+    # This facade delegates rendering to the v2 engine; the retired
+    # options (config:, template overrides in liquid syntax) are ignored
+    # with one warning — migrate to a CitationStyle instance, which
+    # expresses everything the old config.yml did (see the citation
+    # module in relaton-models).
+    class General
+      # The eref `style=` vocabulary metanorma-standoc validates against
+      # (cleanup/xref.rb reads citetemplate.template_raw's keys) — the
+      # 1.x default config's top-level citation styles.
+      CITE_STYLES = %i[
+        author_date author_date_br author date reference_tag title
+        title_reference_tag short
+      ].freeze
+
+      CiteTemplate = Struct.new(:template_raw)
+
+      def initialize(options = {})
+        options = deep_symbolize(options)
+        @lang = options[:language] || options[:lang] || "en"
+        renderer_opts = { lang: @lang, script: options[:script] || "Latn",
+                          labels: options[:i18nhash] || {} }
+        # A flavor names its own CitationStyle instance (name or YAML path)
+        options[:style] and renderer_opts[:style] = options[:style]
+        @renderer = Iso690::Renderer.new(**renderer_opts)
+        @renderer_opts = renderer_opts
+        @renderers_by_lang = {}
+        warn_general_config if options[:config]
+      end
+
+      # A collection's items may each declare their own language; the
+      # renderer localizes by language, so a reference renders in its
+      # own language within a document of another (the 1.x plateau
+      # per-reference behaviour)
+      def renderer_for(lang)
+        return @renderer if lang.nil? || lang == @lang
+
+        @renderers_by_lang[lang] ||= Iso690::Renderer.new(
+          **@renderer_opts.merge(lang: lang,
+                                 script: %w(ja ko zh).include?(lang) ? nil : "Latn")
+        )
+      end
+
+      def render(model, embedded: false, **opts)
+        model = to_model(model)
+        text =
+          if model.respond_to?(:formattedref) && model.formattedref
+            model.formattedref.content
+          else
+            # a collection's items may each declare their own language;
+            # the single-item render routes like render_all does
+            renderer_for(Array(model.language).first).render(model, **opts)
+          end
+        embedded ? text : "<formattedref>#{text}</formattedref>"
+      end
+
+      def citation(model, **opts)
+        @renderer.citation(to_model(model), **opts)
+      end
+
+      # isodoc's references rendering: renderings[id][:formattedref]
+      # feeds the <formattedref> of every bibitem (isodoc
+      # presentation_function/refs.rb).
+      def render_all(bib, type: "author-date")
+        items = facade_bibitems(bib) or return nil
+        disambiguators = date_disambiguators(items)
+        items.each_with_object({}).with_index do |(item, m), i|
+          renderer = renderer_for(Array(item.language).first)
+          ref = begin
+            renderer.render(item, disambiguator: disambiguators[item.id])
+          rescue ::Relaton::Render::Unrenderable
+            next
+          end
+          m[item.id] = {
+            author: renderer.in_text_author(item),
+            date: renderer.disambiguated_date(item,
+                                              disambiguator:
+                                                disambiguators[item.id]),
+            citation: citation_renderings(item, ref, renderer,
+                                          type: type,
+                                          disambiguator:
+                                            disambiguators[item.id]),
+            formattedref: terminate_reference(ref, item),
+          }
+        end
+      end
+
+      # isodoc's styled citations consume these keys: the default is the
+      # authoritative identifier; the short cite is the style's citation,
+      # falling back to the reference rendering (the 1.x short-cite)
+      FIRST_DELIM = '<span class="fmt-first-biblio-delim"/>'
+
+      # A creator list alone (flavors' document-history name forms)
+      def creator_names(item)
+        @renderer.render_creators(to_model(item))
+      end
+
+      # render_all feeds the bibliography list: a reference not ending
+      # in the biblio terminator takes one (the 1.x render1 behaviour;
+      # single-item render stays verbatim)
+      # The batch disambiguation suffixes ("2022a"): items sharing a
+      # principal-creator surname and a date take alphabetical
+      # suffixes in bibliography order (the 1.x
+      # disambig_author_date_citations)
+      def date_disambiguators(items)
+        keys = items.filter_map do |item|
+          begin
+            [item.id, @renderer.author_key(item),
+             @renderer.citation(item)]
+          rescue ::Relaton::Render::Unrenderable
+            nil
+          end
+        end
+        groups = keys.group_by { |_, author, cite| [author, cite] }
+        suffixes = {}
+        groups.each_value do |group|
+          next if group.size < 2
+
+          group.each_with_index do |(id, _author, _cite), i|
+            suffixes[id] = ("a".ord + i).chr
+          end
+        end
+        suffixes
+      end
+
+      # The bibliography terminator; flavors override to suppress it
+      # (1.x use_terminator?), receiving the item it terminates. A
+      # reference closing on a footnote closes the sentence before it:
+      # the terminator lands ahead of the trailing fn.
+      FOOTNOTE_CLOSE = "</fn>".freeze
+
+      def terminate_reference(ref, _item = nil)
+        return ref if ref.empty? || ref.rstrip.end_with?(".")
+
+        if (fn = ref.rindex("<fn"))
+          head = ref[0...fn].rstrip
+          return ref if head.empty?
+
+          sep = head.end_with?(".") ? "" : "."
+          return "#{head}#{sep}#{ref[fn..]}"
+        end
+
+        "#{ref}."
+      end
+
+      # The style-keyed capsule ("citation" => { author_date: "..." })
+      # is what isodoc's styled erefs consume (their style attribute
+      # names the key)
+      def citation_renderings(item, ref, renderer, type: "author-date",
+                              disambiguator: nil)
+        # A style declaring shortFromReference takes the reference form
+        # with the first-biblio marker (the 1.x citetemplate short)
+        short = if renderer.short_from_reference?
+                  renderer.render_short(item, FIRST_DELIM)
+                else
+                  short = renderer.citation(item)
+                  short.empty? ? renderer.render_short(item, FIRST_DELIM) :
+                    short
+                end
+
+        title = renderer.title_citation(item)
+
+        {
+          full: ref,
+          default: item.docidentifier.first&.content.to_s,
+          short: short,
+          author_date: renderer.citation(item,
+                                         disambiguator: disambiguator),
+          author_date_br: renderer.bracketed_date_citation(
+            item, disambiguator: disambiguator,
+          ),
+          author: renderer.in_text_author(item),
+          date: renderer.disambiguated_date(item),
+          # reference_tag: no rendering — isodoc falls back to the
+          # biblio-tag anchor (the 1.x nil)
+          title: title,
+          title_reference_tag: title,
+        }
+      end
+
+      # isodoc's pref_ref_code_parse reads
+      # data[:authoritative_identifier] to build biblio-tag identifiers
+      # (presentation_function/docid.rb). Returns [data, self] as the
+      # 1.x signature did.
+      def parse(doc)
+        [{ authoritative_identifier: facade_docids(doc) }, self]
+      end
+
+      def citetemplate
+        CiteTemplate.new(CITE_STYLES.to_h { |s| [s, ""] })
+      end
+
+      private
+
+      # 1.x accepted a <references> XML string (isodoc strips namespaces
+      # before building it) or a ready array of models.
+      def facade_bibitems(bib)
+        case bib
+        when Array then bib
+        when String
+          require "moxml"
+          root = Moxml.parse(bib).root
+          return nil unless root&.name == "references"
+
+          klass = facade_bibitem_class
+          root.xpath("./bibitem").filter_map { |b| klass.from_xml(b.to_xml) }
+        end
+      end
+
+      def facade_bibitem_class
+        Relaton::Bib::Bibitem
+      rescue NameError
+        Relaton::Bib::Item
+      end
+
+      # 1.x Parse#authoritative_identifier semantics: the
+      # primary/language cascade, the excluded types (URN, DOI, ISBN...)
+      # dropped, and scoped ids (biblio-tag duplicates, IEEE trademark)
+      # demoted within their type group. isodoc hands the live bibitem
+      # node with the document's default namespace still attached, so the
+      # element queries must be namespace-agnostic.
+      EXCLUDED_ID_TYPES = %w(METANORMA METANORMA-ORDINAL AUTHOR-DATE TITLE
+                             URN ISO-REFERENCE ISSN ISBN DOI).freeze
+
+      # Flavors extend the authoritative-identifier exclusions by
+      # overriding this reader on their facade subclass
+      def excluded_id_types
+        EXCLUDED_ID_TYPES
+      end
+
+      def facade_docids(doc)
+        node = doc
+        unless node.respond_to?(:xpath)
+          require "moxml"
+          node = Moxml.parse(doc.to_s).root
+        end
+        return [] unless node
+
+        ids = node.xpath("./*[local-name() = 'docidentifier']")
+        ids = facade_scope_filter(ids)
+        out = nil
+        [
+          ->(x) { x["language"] == @lang && x["primary"] },
+          ->(x) { x["primary"] },
+          ->(x) { x["language"] == @lang },
+          ->(_x) { true },
+        ].each do |p|
+          out = ids.select do |x|
+            p.call(x) && !excluded_id_types.include?(facade_id_type(x))
+          end
+          out.empty? or break
+        end
+        # esc-wrapped: isodoc's l10n skips the tags, leaving a standard
+        # identifier's punctuation verbatim (the 1.x Parse behaviour)
+        out.map { |x| "<esc>#{x.text.strip}</esc>" }
+      end
+
+      def facade_id_type(id)
+        t = id["type"] or return nil
+        m = /\A(ISBN|ISSN)\..*/i.match(t) or return t.upcase
+        m[1].upcase
+      end
+
+      def facade_scope_filter(ids)
+        ids.detect { |i| i["scope"] } or return ids
+        ids.group_by { |i| i["type"] }.flat_map do |type, group|
+          grouped = group.group_by { |i| i["scope"] }
+          if type == "IEEE" then grouped["trademark"] || grouped[nil] || []
+          else grouped[nil] || []
+          end
+        end
+      end
+
+      # 1.x accepted string- and symbol-keyed options alike (the old
+      # engine deep-symbolized via metanorma-utils); isodoc's bibrenderer
+      # and every flavor still pass string keys.
+      def deep_symbolize(value)
+        case value
+        # isodoc i18n hashes carry nil keys (an unlabeled punct slot)
+        when Hash then value.to_h { |k, v| [k&.to_sym, deep_symbolize(v)] }
+        when Array then value.map { |v| deep_symbolize(v) }
+        else value
+        end
+      end
+
+      # 1.x render/citation accepted a bibitem XML string; the Iso690
+      # engine consumes model instances.
+      def to_model(model)
+        return model unless model.is_a?(String)
+
+        klass = facade_bibitem_class
+        klass.from_xml(model)
+      end
+
+      def warn_general_config
+        return if General.config_warned
+
+        General.config_warned = true
+        warn "relaton-render: Relaton::Render::General is a compatibility " \
+             "facade over the v2 engine; the liquid config option is " \
+             "retired and ignored. Author a CitationStyle instance " \
+             "(relaton-models citation module) for custom rendering."
+      end
+
+      class << self
+        attr_accessor :config_warned
+      end
+    end
+  end
+end