relaton 3.0.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/jcgm/meetings_parser.rb

@@ -0,0 +1,175 @@
+require "fileutils"
+require "yaml"
+require "date"
+
+module Relaton::Jcgm
+  # Harvests JCGM meeting proceedings from the `jcgm/meetings-{en,fr}` dirs of a
+  # local checkout of metanorma/bipm-data-outcomes and writes one Relaton YAML
+  # per meeting into `<output>/jcgm/meeting/<num>.yaml`, indexing each by its
+  # `Pubid::Jcgm` meeting identifier.
+  #
+  # This is the JCGM-only slice of `Relaton::Bipm::DataOutcomesParser`: JCGM
+  # meetings carry no sub-resolutions and no parts, so the resolution/part
+  # machinery is dropped. The meeting docnumber's ordinal ("11st"/"12nd"/"13rd"
+  # — a naive last-digit rule with no teens exception) is delegated to
+  # `Pubid::Jcgm::Identifiers::Meeting.ordinal`, the single source of truth, so
+  # the printed form round-trips through pubid.
+  class MeetingsParser
+    SOURCE_ROOT = "bipm-data-outcomes".freeze
+    BODY = "JCGM".freeze
+    GH_SRC = "https://raw.githubusercontent.com/metanorma/bipm-data-outcomes/".freeze
+
+    def self.parse(data_fetcher)
+      new(data_fetcher).parse
+    end
+
+    def initialize(data_fetcher)
+      @data_fetcher = data_fetcher
+    end
+
+    def parse
+      Dir[File.join(SOURCE_ROOT, "jcgm", "meetings-en", "*.{yml,yaml}")].sort.each do |en_file|
+        fetch_meeting en_file
+      end
+    end
+
+    private
+
+    def fetch_meeting(en_file)
+      en, fr, fr_file = read_files en_file
+      md_en = en["metadata"]
+      md_fr = fr && fr["metadata"]
+      num = md_en["identifier"].to_s
+      date = md_en["date"].to_s
+      docnum = meeting_docnum num, date
+      id = "JCGMMeeting#{num}#{::Date.parse(date).year}"
+
+      outdir = File.join @data_fetcher.output, "jcgm", "meeting"
+      FileUtils.mkdir_p outdir
+      path = File.join outdir, "#{num}.#{@data_fetcher.ext}"
+
+      item = ItemData.new(**meeting_hash(md_en, md_fr, num, id, docnum,
+                                         meeting_links(en_file, fr_file), en["pdf"]))
+      @data_fetcher.write_file path, item
+      @data_fetcher.add_to_index docnum, path
+    end
+
+    # Read English + French metadata (the French file mirrors the path with
+    # `en` -> `fr`). `Date`/`Time` are permitted because the source may write
+    # an unquoted `date:` scalar; the date is coerced to a string downstream.
+    def read_files(en_file)
+      fr_file = en_file.sub "en", "fr"
+      en = load_yaml en_file
+      fr = File.exist?(fr_file) ? load_yaml(fr_file) : nil
+      [en, fr, (fr && fr_file)]
+    end
+
+    def load_yaml(file)
+      YAML.safe_load File.read(file, encoding: "UTF-8"), permitted_classes: [Date, Time]
+    end
+
+    # "JCGM 17th Meeting (2012)" — ordinal via pubid's Meeting.ordinal.
+    def meeting_docnum(num, date)
+      year = ::Date.parse(date).year
+      "#{BODY} #{::Pubid::Jcgm::Identifiers::Meeting.ordinal(num)} Meeting (#{year})"
+    end
+
+    def meeting_hash(md_en, md_fr, num, id, docnum, src, pdf) # rubocop:disable Metrics/MethodLength
+      {
+        id: id,
+        type: "proceedings",
+        title: titles(md_en, md_fr),
+        place: [Relaton::Bib::Place.new(city: "Paris")],
+        date: date(md_en["date"]),
+        docidentifier: docids(docnum),
+        docnumber: docnum,
+        source: links(md_en, md_fr, src, pdf),
+        language: %w[en fr],
+        script: ["Latn"],
+        contributor: [bipm_contrib],
+        ext: ext(num),
+      }
+    end
+
+    def titles(md_en, md_fr)
+      result = []
+      result << title(md_en["title"], "en") if md_en && md_en["title"]
+      result << title(md_fr["title"], "fr") if md_fr && md_fr["title"]
+      result
+    end
+
+    def title(content, language)
+      content = content.dup
+      content.sub!(/(\d+)(e)/, '\1<sup>\2</sup>') if language == "fr"
+      Relaton::Bib::Title.new(content: content, language: language, script: "Latn")
+    end
+
+    def date(at)
+      at = at.to_s
+      return [] if at.empty?
+
+      [Relaton::Bib::Date.new(type: "published", at: at)]
+    end
+
+    # en: "JCGM 17th Meeting (2012)"; fr: "JCGM 17<sup>e</sup> réunion (2012)";
+    # plus the combined "en / fr" form (mirrors BIPM's create_meeting_docids).
+    def docids(en_id)
+      fr_id = en_id.sub(/(\d+)(?:st|nd|rd|th)/, '\1e').sub("Meeting", "réunion")
+      fr_id_sup = fr_id.sub(/(\d+)(e)/, '\1<sup>\2</sup>')
+      [
+        docid(en_id, language: "en"),
+        docid(fr_id_sup, language: "fr"),
+        docid("#{en_id} / #{fr_id_sup}"),
+      ]
+    end
+
+    def docid(content, language: nil)
+      attrs = { content: content, type: "BIPM", primary: true }
+      attrs.merge!(language: language, script: "Latn") if language
+      Docidentifier.new(**attrs)
+    end
+
+    def links(md_en, md_fr, src, pdf)
+      links = []
+      links << uri("citation", md_en["url"], "en") if md_en && md_en["url"]
+      links << uri("citation", md_fr["url"], "fr") if md_fr && md_fr["url"]
+      Array(pdf).each { |p| links << Relaton::Bib::Uri.new(type: "pdf", content: p) }
+      links + Array(src)
+    end
+
+    def uri(type, content, language)
+      Relaton::Bib::Uri.new(type: type, content: content, language: language, script: "Latn")
+    end
+
+    # en/fr `src` links back to the source files in bipm-data-outcomes.
+    def meeting_links(en_file, fr_file)
+      { "en" => en_file, "fr" => fr_file }.filter_map do |lang, file|
+        next unless file
+
+        src = GH_SRC + file.split("/")[-3..].unshift("main").join("/")
+        Relaton::Bib::Uri.new(type: "src", content: src, language: lang, script: "Latn")
+      end
+    end
+
+    def ext(num)
+      Ext.new(doctype: Doctype.new(content: "meeting-report"),
+              structuredidentifier: StructuredIdentifier.new(docnumber: num))
+    end
+
+    def bipm_contrib
+      names = [
+        Relaton::Bib::TypedLocalizedString.new(content: "International Bureau of Weights and Measures", language: "en", script: "Latn"),
+        Relaton::Bib::TypedLocalizedString.new(content: "Bureau international des poids et mesures", language: "fr", script: "Latn"),
+      ]
+      org = Relaton::Bib::Organization.new(
+        name: names,
+        abbreviation: Relaton::Bib::LocalizedString.new(content: "BIPM", language: "fr", script: "Latn"),
+        uri: [Relaton::Bib::Uri.new(content: "www.bipm.org")],
+      )
+      Relaton::Bib::Contributor.new(
+        organization: org,
+        role: [Relaton::Bib::Contributor::Role.new(type: "publisher")],
+      )
+    end
+  end
+end

data/lib/relaton/jcgm/processor.rb

@@ -0,0 +1,71 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Jcgm
+    class Processor < Core::Processor
+      attr_reader :idtype
+
+      def initialize
+        @short = :relaton_jcgm
+        @prefix = "JCGM"
+        @defaultprefix = %r{^JCGM\s}
+        @idtype = "JCGM"
+        @datasets = %w[bipm-data-outcomes]
+        # Prefixes come from pubid (Pubid::Jcgm.prefixes => ["JCGM"]); see
+        # Core::Processor#prefixes.
+        @pubid_flavor = :Jcgm
+      end
+
+      # @param code [String]
+      # @param date [String, nil] year
+      # @param opts [Hash]
+      # @return [Relaton::Jcgm::ItemData, nil]
+      def get(code, date, opts)
+        require_relative "../jcgm"
+        Bibliography.get(code, date, opts)
+      end
+
+      #
+      # Harvest the JCGM meeting proceedings from metanorma/bipm-data-outcomes.
+      # Used to (re)generate the relaton-data-jcgm repo; the curated static guides
+      # are indexed by that repo's crawler via DataFetcher#add_to_index.
+      #
+      # @param [String] source source name ("bipm-data-outcomes")
+      # @param [Hash] opts
+      #
+      def fetch_data(source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(source, **opts)
+      end
+
+      # @param xml [String]
+      # @return [Relaton::Jcgm::ItemData]
+      def from_xml(xml)
+        require_relative "../jcgm"
+        Item.from_xml xml
+      end
+
+      # @param yaml [String]
+      # @return [Relaton::Jcgm::ItemData]
+      def from_yaml(yaml)
+        require_relative "../jcgm"
+        Item.from_yaml yaml
+      end
+
+      # Returns hash of XML grammar
+      # @return [String]
+      def grammar_hash
+        require_relative "../jcgm"
+        @grammar_hash ||= ::Relaton::Jcgm.grammar_hash
+      end
+
+      #
+      # Remove index file
+      #
+      def remove_index_file
+        require_relative "../jcgm"
+        Relaton::Index.find_or_create(:jcgm, url: true, file: "#{INDEXFILE}.yaml").remove_file
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/relation.rb

@@ -0,0 +1,9 @@
+require_relative "item_base"
+
+module Relaton
+  module Jcgm
+    class Relation < Bib::Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/jcgm/structured_identifier.rb

@@ -0,0 +1,40 @@
+module Relaton
+  module Jcgm
+    # A single structured identifier (e.g. `{ docnumber: "17" }`). JCGM records
+    # carry this as a single mapping, not a sequence, so it overrides the base
+    # `Bib::Ext#structuredidentifier` collection (mirrors BIPM's model).
+    class StructuredIdentifier < Lutaml::Model::Serializable
+      attribute :docnumber, :string
+      attribute :part, :string
+      attribute :appendix, :string
+
+      xml do
+        map_element "docnumber", to: :docnumber
+        map_element "part", to: :part
+        map_element "appendix", to: :appendix
+      end
+
+      # `Bib::ItemData#ext_to_all_parts!` / `#ext_remove_date` broadcast to
+      # `ext.structuredidentifier` just as they do to `docidentifier`, so these
+      # three are as required here as on the docidentifier. This class descends
+      # from `Lutaml::Model::Serializable` rather than `Bib::StructuredIdentifier`
+      # (JCGM stores a single mapping, not a sequence), so the methods were not
+      # merely unimplemented — they were absent, and the broadcast raised
+      # `NoMethodError` on every JCGM item.
+      #
+      # JCGM structured identifiers carry no date field, so `remove_date!` has
+      # nothing to strip and is a deliberate no-op; the year lives on the
+      # docidentifier's pubid, which `Docidentifier#remove_date!` handles.
+
+      def remove_part!
+        self.part = nil
+      end
+
+      def remove_date!; end
+
+      def to_all_parts!
+        remove_part!
+      end
+    end
+  end
+end

data/lib/relaton/jcgm/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Jcgm
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-jcgm".freeze
+    end
+  end
+end

data/lib/relaton/jcgm.rb

@@ -0,0 +1,24 @@
+require "relaton/bib"
+require "relaton/index"
+require "pubid"
+require_relative "version"
+require_relative "jcgm/util"
+require_relative "jcgm/docidentifier"
+require_relative "jcgm/item"
+require_relative "jcgm/bibitem"
+require_relative "jcgm/bibdata"
+require_relative "jcgm/bibliography"
+
+module Relaton
+  module Jcgm
+    INDEXFILE = "index-v1".freeze
+
+    class Error < StandardError; end
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/jis/bibliography.rb

@@ -16,16 +16,15 @@ module Relaton
       # @param [String] code JIS document code
       # @param [String, nil] year JIS document year
       #
-      # @return [Relaton::Jis::HitCollection, nil] search result, or nil when
-      #   the reference cannot be parsed
+      # @return [Relaton::Jis::HitCollection] search result
+      #
+      # @raise [Pubid::Errors::ParseError] when the reference is not a JIS
+      #   identifier, so a malformed reference is not reported as "not found"
       #
       def search(code, year = nil)
         pubid = ::Pubid::Jis::Identifier.parse code
         pubid.year ||= year.to_i if year
         HitCollection.new pubid
-      rescue StandardError => e
-        Util.warn "Unable to parse `#{code}` with pubid: #{e.message}"
-        nil
       end
 
       #
@@ -38,15 +37,14 @@ module Relaton
       #
       # @return [Relaton::Jis::Item, nil] JIS document
       #
-      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      # @raise [Pubid::Errors::ParseError] when the reference is not a JIS
+      #   identifier
+      #
+      def get(ref, year = nil, opts = {}) # rubocop:disable Metrics/AbcSize
         code = ref.sub(/\s\((all parts|規格群)\)/, "")
         opts[:all_parts] ||= !$1.nil?
         Util.info "Fetching from webdesk.jsa.or.jp ...", key: ref
         hits = search(code, year)
-        unless hits
-          hint [], ref, year
-          return
-        end
         result = opts[:all_parts] ? hits.find_all_parts : hits.find
         if result.is_a? Bib::ItemData
           Util.info "Found: `#{result.docidentifier[0].content}`", key: ref

data/lib/relaton/jis/data_fetcher.rb

@@ -19,23 +19,17 @@ module Relaton
         Util.error msg
       end
 
+      # The pubid index-v2, the only index a crawl writes. relaton-data-jis
+      # rebuilds the legacy string index-v1 from data/ in its own crawler.
       def index
-        @index ||= Relaton::Index.find_or_create :jis, file: "#{INDEXFILE}.yaml"
-      end
-
-      # Pubid-based index built in parallel with the legacy string index. The
-      # pool keys by type, so requesting a second :jis index with a different
-      # file evicts the v1 Type from the pool, but we keep our own reference in
-      # @index, so both stay live for the duration of the crawl.
-      def index_v2
-        @index_v2 ||= Relaton::Index.find_or_create(
-          :jis, file: "#{INDEXFILE_V2}.yaml", pubid_class: ::Pubid::Jis::Identifier
+        @index ||= Relaton::Index.find_or_create(
+          :jis, file: "#{INDEXFILE}.yaml", pubid_class: ::Pubid::Jis::Identifier
         )
       end
 
       # Parse a primary docidentifier string into a pubid identifier; nil (with
       # a warning) if pubid can't parse it, so a single bad id never aborts the
-      # crawl or corrupts index-v2.
+      # crawl or corrupts the index.
       def pubid(id)
         ::Pubid::Jis::Identifier.parse id
       rescue StandardError => e
@@ -89,7 +83,6 @@ module Relaton
         resp = agent.get "#{URL}W11M0070/index"
         parse_page resp
         index.save
-        index_v2.save
         report_errors
       end
 
@@ -163,17 +156,26 @@ module Relaton
         return unless bib
 
         id = bib.docidentifier.find(&:primary).content
-        file = output_file id
         @mutex.synchronize do
+          # Two distinct ids can sanitize to one filename, in which case the
+          # second document used to be dropped. Give it a path of its own; a
+          # genuine duplicate (same id) still resolves to one path and is still
+          # skipped. Inside the lock: unique_output_file reserves state.
+          file = unique_output_file id
+          # A reserved path only ever belongs to one id, so a hit here is the
+          # same document again. Checked FIRST: a disambiguated path stays !=
+          # output_file forever, so gating this on that comparison would make a
+          # repeat of a disambiguated id skip the skip and overwrite itself.
           if @files.include?(file)
             Util.warn "File #{file} already exists. Duplication URL: #{url}"
-          else
-            @files << file
-            File.write file, serialize(bib), encoding: "UTF-8"
-            index.add_or_update id, file
-            pid = pubid id
-            index_v2.add_or_update pid, file if pid
+            next
           end
+
+          Util.warn "File #{output_file id} already exists; writing #{file} instead" if file != output_file(id)
+          @files << file
+          File.write file, serialize(bib), encoding: "UTF-8"
+          pid = pubid id
+          index.add_or_update pid, file if pid
         end
       end
     end

data/lib/relaton/jis/docidentifier.rb

@@ -2,19 +2,118 @@
 
 module Relaton
   module Jis
+    # A JIS document identifier, backed by `Pubid::Jis`.
+    #
+    # `content` stays the source string for serialization. `#pubid` parses it
+    # into a `Pubid::Jis::Identifier`, and the three mutators change that pubid
+    # and render it back into `content`. There is no string surgery.
+    #
+    # The parse is lazy and soft, as in `Relaton::Itu::Docidentifier`: it runs
+    # on first use, and content that does not parse gives a nil `#pubid`, with
+    # no log. The all-parts form "JIS C 0364 (all parts)" does not parse, and
+    # a cached all-parts item is deserialized again, so a log would be noise.
+    # With a nil `#pubid` the mutators do nothing.
+    #
+    # ## Rendering rules
+    #
+    # - The `JIS ` publisher is rendered only when the source has it. pubid
+    #   always adds it, but the source spells a technical report
+    #   `TR Z 0010:2008`.
+    # - Everything else is the pubid canonical spelling. pubid does not keep the
+    #   source spelling `AMENDMENT 1` / `EXPLANATION 1`, so a changed supplement
+    #   renders as `AMD 1` / `EXPL 1`, the form that the index uses.
+    # - The all-parts marker is ` (all parts)`, as in
+    #   `Relaton::Jis::StructuredIdentifier`, not the pubid `（規格群）`.
+    #   `Bibliography.get` accepts both as a reference.
     class Docidentifier < Bib::Docidentifier
+      # Capture the inherited content setter before the override, so that a
+      # mutation can write the rendered string back and keep the changed pubid.
+      # `content=` would clear it, and a new parse of "… (all parts)" fails.
+      alias_method :store_content, :content=
+
+      def content=(value)
+        super
+        @pubid = nil
+        @pubid_parsed = false
+      end
+
+      # @return [Pubid::Jis::Identifier, nil] nil if the content does not parse
+      def pubid
+        return @pubid if @pubid_parsed
+
+        @pubid_parsed = true
+        @pubid = parse
+      end
+
+      # Removes every part level: `JIS C 0364-2-21:1999` -> `JIS C 0364:1999`.
+      # For a supplement, the part is on its base.
       def remove_part!
-        content&.sub!(/-\d+/, "")
+        return unless pubid
+
+        change_document { |id| id.exclude(:parts) }
+        refresh_content!
       end
 
+      # Removes the year and the reaffirmation marker:
+      # `JIS C 9901:2019R` -> `JIS C 9901`.
+      #
+      # For a supplement, only the base loses its date, and the supplement keeps
+      # its own year: `JIS B 3700-101:1996/CORRIGENDUM 1:2002` ->
+      # `JIS B 3700-101/CORRIGENDUM 1:2002`. pubid renders a supplement without
+      # a year as `…/CORRIGENDUM 1:`, with a trailing colon.
       def remove_date!
-        content&.sub!(/:\d{4}/, "")
+        return unless pubid
+
+        change_document { |id| id.exclude(:year, :reaffirmed) }
+        refresh_content!
       end
 
+      # `JIS C 0364-2-21:1999` -> `JIS C 0364 (all parts)`, with `all_parts`
+      # set on `#pubid`.
       def to_all_parts!
-        remove_part!
-        remove_date!
-        self.content = "#{content} (all parts)" if content
+        return if !pubid || pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy;
+        # `change_document` itself never mutates in place (it only reassigns
+        # `@pubid`), but this stays correct even if that changes.
+        original = @pubid.exclude
+        change_document { |id| id.exclude(:parts, :year, :reaffirmed) }
+        document = render(@pubid)
+        @pubid = original.to_all_parts
+        store_content "#{document} (all parts)"
+      end
+
+      private
+
+      def parse
+        return unless content
+
+        require "pubid"
+        ::Pubid::Jis::Identifier.parse content.to_s
+      rescue LoadError, StandardError
+        nil
+      end
+
+      # Applies the change to the document identifier: the pubid itself, or the
+      # base of a supplement. Copy on write, as `Pubid::Identifier#exclude` is:
+      # an argument-less `exclude` copies the supplement, so a pubid that a
+      # caller got before the mutation does not change.
+      def change_document(&)
+        @pubid = document_changed(@pubid, &)
+      end
+
+      def document_changed(id, &)
+        return yield(id) unless id.is_a?(::Pubid::Jis::SupplementIdentifier)
+
+        id.exclude.tap { |copy| copy.base &&= document_changed(copy.base, &) }
+      end
+
+      def refresh_content!
+        store_content render(@pubid)
+      end
+
+      def render(id)
+        id.to_s(with_publisher: content.to_s.start_with?("JIS"))
       end
     end
   end

data/lib/relaton/jis/hit.rb

@@ -15,24 +15,34 @@ module Relaton
         new hit, collection
       end
 
+      # The edition aspects. {HitCollection} picks the edition, so a match
+      # ignores them. The reaffirmation mark `R` goes with the year:
+      # `exclude(:year)` keeps it, and `JIS L 4107` must match
+      # `JIS L 4107:2000R`.
+      EDITION = %i[year reaffirmed].freeze
+
       #
       # Check if the hit matches the collection's reference.
       #
-      # The candidate must be the same document type (so a plain standard query
-      # never matches its amendments) and share series and number. Part is
-      # compared only when the reference names a specific part and `all_parts`
-      # is off. Year is filtered separately by {HitCollection}.
+      # The candidate must be the same document type, so a plain standard
+      # query never matches its amendments. The class check is explicit
+      # because pubid's JIS `==` does not compare the class
+      # (`JIS TR X 0014:1999 == JIS X 0014:1999`). The other aspects are
+      # compared, except the edition, and the parts when `all_parts` is on.
+      # This includes the base document of a supplement. The `SYMBOL` is
+      # compared on a standard, but not on a supplement: pubid's supplement
+      # `==` ignores it. {HitCollection#find_by_year} puts the exact printed id
+      # first for that reason.
       #
       # @param [Boolean] all_parts match any part of the document
       #
       # @return [Boolean] true if the hit matches
       #
       def matches?(all_parts: false)
-        cand = pubid
-        return false unless cand && same_document?(cand)
-        return true if all_parts || reference_partless?
+        return false unless pubid.instance_of?(reference.class)
 
-        Array(cand.parts).map(&:to_s) == reference_parts
+        ignore = all_parts ? EDITION + [:parts] : EDITION
+        reference.matches? pubid, ignore: ignore
       end
 
       #
@@ -73,21 +83,6 @@ module Relaton
       def reference
         hit_collection.pubid
       end
-
-      # Same document type, series and number as the reference.
-      def same_document?(cand)
-        cand.instance_of?(reference.class) &&
-          cand.series == reference.series &&
-          cand.number.to_s == reference.number.to_s
-      end
-
-      def reference_partless?
-        reference.parts.nil? || reference.parts.empty?
-      end
-
-      def reference_parts
-        Array(reference.parts).map(&:to_s)
-      end
     end
   end
 end