pubid 2.0.0.pre.alpha.13 → 2.0.0.pre.alpha.14

data/lib/pubid/itu/parser.rb

@@ -125,9 +125,48 @@ module Pubid
         dash >> (letter.repeat(1, 3) >> dot >> digits).as(:range_end)
       end
 
-      # Parts
+      # A "-YYYYMM" approval date — the "200307" of "T-REC-T.4-200307-I" and
+      # "ITU-T T.4-200307". ITU's own edition suffix is short ("-5"), so six
+      # digits that read as a plausible year (19xx/20xx) and month (01-12) are
+      # the date, never a part. A six-digit run that fails either test
+      # ("-200313", "-180001") is still a part, as it was before.
+      rule(:yyyymm_year) { (str("19") | str("20")) >> digit >> digit }
+      rule(:yyyymm_month) do
+        (str("0") >> match["1-9"]) | (str("1") >> match["0-2"])
+      end
+      rule(:yyyymm_shape) { yyyymm_year >> yyyymm_month >> digit.absent? }
+
+      # The status letter that trails the date in a publication id — "I" (in
+      # force) or "S" (superseded). It names the state of the edition, not the
+      # edition, so it is parsed and dropped. "S" is also the Spanish language
+      # suffix, so it is a status ONLY in the full "T-REC-…" id, where ITU
+      # always writes one; after an "ITU-T …-YYYYMM" print form only "I" is,
+      # and "-S" stays the language ("ITU-T Z.100-199911-S").
+      rule(:id_status) do
+        dash >> match["IS"] >> match["A-Za-z0-9"].absent?
+      end
+
+      rule(:print_id_status) do
+        dash >> str("I") >> match["A-Za-z0-9"].absent?
+      end
+
+      rule(:yyyymm_date) do
+        dash >> yyyymm_year.as(:year) >> yyyymm_month.as(:month) >>
+          digit.absent?
+      end
+
+      rule(:id_date) { yyyymm_date >> print_id_status.maybe }
+
+      # Either date spelling of a Recommendation.
+      rule(:document_date) { date_part | id_date }
+
+      # "ITU-T REC T.4", "ITU-T REC-T.4" — the redundant type word of ITU's
+      # own URLs. Not captured: a Recommendation is the default type.
+      rule(:rec_word) { str("REC") >> (space | dash) }
+
+      # Parts. The yyyymm guard keeps the approval date out of the part list.
       rule(:part) do
-        dash >> digits.as(:part)
+        dash >> yyyymm_shape.absent? >> digits.as(:part)
       end
 
       rule(:parts) { part.repeat(0).as(:parts) }
@@ -273,6 +312,7 @@ module Pubid
         itu_prefix >>
           sector >>
           space >>
+          rec_word.maybe >>
           series >> dot >>
           code >>
           range_end.maybe >>
@@ -281,7 +321,7 @@ module Pubid
           series_word.maybe >>
           attachment.maybe >>
           version_part.maybe >>
-          date_part.maybe
+          document_date.maybe
       end
 
       rule(:base_without_series) do
@@ -292,7 +332,7 @@ module Pubid
           code_suffixes >>
           attachment.maybe >>
           version_part.maybe >>
-          date_part.maybe
+          document_date.maybe
       end
 
       # A series-code document — "EMC-5", "MES-2", "QOS-2", "IMPL-8",
@@ -311,12 +351,10 @@ module Pubid
       # The number stays in `code.number`, so `root.number` — the field
       # relaton-index bsearches on — is "5" for EMC-5 and "QKD" for SEC-QKD
       # rather than nil.
-      # The OB guard keeps "ITU-T OB-1" a clean parse failure. Without it the
-      # string reaches Builder#build's Recommendation fallback, whose
-      # validate_ob_no_sector! raises an ArgumentError that escapes
-      # Identifier.parse's Parslet::ParseFailed rescue — turning a rejected
-      # input into a crash for callers. It guards "OB" + dash specifically, so
-      # a genuine two-letter mnemonic starting "OB" would still parse.
+      # The OB guard keeps "ITU-T OB-1" a clean parse failure rather than a
+      # Recommendation of a series "OB" — the Operational Bulletin's series
+      # name. It guards "OB" + dash specifically, so a genuine two-letter
+      # mnemonic starting "OB" would still parse.
       rule(:series_code_body) do
         (str("OB") >> dash).absent? >>
           letter.repeat(2).as(:series) >> dash.as(:series_dash) >>
@@ -365,10 +403,8 @@ module Pubid
       # Builder#build's `combined` branch, which builds a CombinedIdentifier and
       # would silently drop the marker — a clean parse failure is better than a
       # Report that comes back as a Recommendation. The OB guard mirrors
-      # series_code_body's: an Operational Bulletin is cross-bureau, and
-      # "Report ITU-T OB.1" would otherwise route to SpecialPublication (marker
-      # dropped) or hit validate_ob_no_sector!, whose ArgumentError escapes
-      # Identifier.parse's Parslet::ParseFailed rescue.
+      # series_code_body's: "Report ITU-T OB.1" would otherwise route to
+      # SpecialPublication with the marker dropped.
       rule(:report_body) do
         (str("OB") >> dot).absent? >>
           (series >> dot).maybe >>
@@ -535,6 +571,7 @@ module Pubid
         itu_prefix >>
           sector >>
           space >>
+          rec_word.maybe >>
           series >> dot >>
           code >>
           range_end.maybe >>
@@ -543,7 +580,7 @@ module Pubid
           series_word.maybe >>
           attachment.maybe >>
           version_part.maybe >>
-          date_part.maybe >>
+          document_date.maybe >>
           language.maybe
       end
 
@@ -556,24 +593,71 @@ module Pubid
           code_suffixes >>
           attachment.maybe >>
           version_part.maybe >>
-          date_part.maybe >>
+          document_date.maybe >>
+          language.maybe
+      end
+
+      # ITU's publication id — "T-REC-T.4-200307-I",
+      # "R-REC-BO.1130-5-202602-I": <sector>-REC-<number>[-<edition>]-<YYYYMM>
+      # [-<status>], the name ITU gives each edition in its URLs and PDF
+      # files. It builds the plain Recommendation it names and renders in the
+      # print form ("ITU-T T.4 (07/2003)"). The date is required: without it
+      # the string names no edition. No other rule starts with a bare sector
+      # letter, so the slot is free.
+      rule(:publication_id) do
+        sector >> dash >> str("REC") >> dash >>
+          series >> dot >> code >> yyyymm_date >> id_status.maybe >>
           language.maybe
       end
 
+      # The Radio Regulations — "ITU-R RR", "ITU-R RR (2020)", and the URL
+      # spelling "ITU-R RR-2020". Always ITU-R. The trailing any.absent? is
+      # load-bearing: PEG ordered choice never re-enters the alternation once
+      # an alternative succeeds, so a partial match on "ITU-R RR.1" must fail
+      # here and fall through to with_series.
+      rule(:radio_regulations) do
+        itu_prefix >> str("R").as(:sector) >> space >>
+          str("RR").as(:radio_regulations) >>
+          (date_part | (dash >> digit.repeat(4, 4).as(:year))).maybe >>
+          language.maybe >> any.absent?
+      end
+
       # OB (Operational Bulletin) — Special Publication.
-      # OB is a cross-bureau ITU publication; sector, when present in legacy
-      # strings like "ITU-T OB.1096", is silently dropped by the builder.
+      # OB is a cross-bureau ITU publication. The TSB spelling carries a
+      # sector ("ITU-T OB.1096 (2016)"); the builder keeps it, and it renders
+      # back, but it is not part of the bulletin's identity.
       rule(:ob_series) { str("OB").as(:series) }
 
       rule(:ob_dot_body) { dot >> number }
       rule(:ob_no_body) { space >> str("No.") >> space >> number }
+      # "ITU OB 1000" — metanorma-itu's docidentifier ("Annex to ITU OB %").
+      # Accepted as an input spelling only; it renders "ITU OB No. 1000", the
+      # form ITU's own bulletin site uses.
+      rule(:ob_bare_body) { space >> number }
+
+      # The date as a bulletin prints it — "ITU-T OB.1096 - 15.III.2016": day,
+      # Roman month, year. The months are tried longest first, because PEG
+      # takes the first alternative that matches and "I" would otherwise win
+      # on "III"; "XIII" matches "XII", then fails on the required dot.
+      rule(:roman_month) do
+        %w[XII XI X IX VIII VII VI V IV III II I]
+          .map { |m| str(m) }.reduce(:|)
+      end
+
+      rule(:ob_roman_date) do
+        str(" - ") >> digit.repeat(2, 2).as(:day) >> dot >>
+          roman_month.as(:roman_month) >> dot >>
+          digit.repeat(4, 4).as(:year)
+      end
+
+      rule(:ob_date) { date_part | ob_roman_date }
 
       rule(:ob_with_sector) do
         itu_prefix >>
           (sector >> space).maybe >>
           ob_series >>
-          (ob_dot_body | ob_no_body) >>
-          date_part.maybe >>
+          (ob_dot_body | ob_no_body | ob_bare_body) >>
+          ob_date.maybe >>
           language.maybe
       end
 
@@ -585,7 +669,7 @@ module Pubid
           str("Operational Bulletin").as(:_op_bull) >>
           space >> str("No.") >> space >>
           number >>
-          date_part.maybe >>
+          ob_date.maybe >>
           language.maybe
       end
 
@@ -671,13 +755,15 @@ module Pubid
           handbook |
           numeric_question |
           letter_question |
+          radio_regulations |
           with_series |
           contribution |
           # Unreachable earlier: special_publication needs the literal "OB",
           # handbook/numeric_question need leading digits, letter_question
           # needs series >> dot, and contribution needs "-C" after the series.
           series_code_identifier |
-          without_series
+          without_series |
+          publication_id
       end
 
       # Common-text form: an ITU identifier followed by "| ISO/IEC ...".

data/lib/pubid/itu/urn_generator.rb

@@ -16,7 +16,10 @@ module Pubid
       def generate_base_urn
         parts = ["urn", "itu"]
 
-        if identifier.sector
+        # An Operational Bulletin is cross-bureau: its sector is a spelling,
+        # not identity (see SpecialPublication#==), so it stays out of the URN.
+        if identifier.sector &&
+            !identifier.is_a?(Identifiers::SpecialPublication)
           sector = identifier.sector.to_s
           parts << sector.to_s.downcase
         else
@@ -56,7 +59,11 @@ module Pubid
 
         if identifier.date
           date = identifier.date
-          if date&.year && date.month
+          # Only an Operational Bulletin's printed date carries a day
+          # ("15.III.2016"); it is in `==`, so it must reach the URN too.
+          if date&.year && date.month && date.day
+            parts << "#{date.day}/#{date.month}/#{date.year}"
+          elsif date&.year && date.month
             parts << "#{date.month}/#{date.year}"
           elsif date&.year
             parts << date.year.to_s

data/lib/pubid/jcgm.rb

@@ -35,7 +35,7 @@ module Pubid
       parser = Parser.new
       builder = Builder.new
 
-      parsed = parser.parse(identifier)
+      parsed = Pubid::Parg::Backend.parse(:jcgm, identifier)
       builder.build(parsed)
     end
 

data/lib/pubid/jis/identifier.rb

@@ -174,7 +174,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:jis, identifier)
         Builder.build(parsed)
       end
     end

data/lib/pubid/nist/builder.rb

@@ -87,6 +87,7 @@ module Pubid
         # Note: :base_portion is lost during parser merge, so check for supplement indicators
         if parsed_hash[:supplement_date_range] || parsed_hash[:supplement_slash_year] ||
             parsed_hash[:supplement_month_year] || parsed_hash[:supplement_year] ||
+            parsed_hash[:supplement_empty] ||
             parsed_hash[:supplement] || parsed_hash[:base_portion]
           return build_circular_supplement(parsed_hash)
         end

data/lib/pubid/nist/identifiers/base.rb

@@ -696,14 +696,25 @@ module Pubid
           result += "#{vol_str}n#{issue_number.number}"
         end
 
-        # Use edition component - NO space before edition in MR format (per NIST spec)
-        result += edition.to_s if edition
+        # With a number, the edition glues to it per the NIST spec
+        # ("800-53r5"); series-only editions take a dot separator
+        # ("NBS.CIRC.e2" — the attested raw spelling; testsuite#5 C4/C5).
+        result += if edition
+                    number ? edition.to_s : ".#{edition}"
+                  else
+                    ""
+                  end
 
         # Use version_component
         result += version_component.to_s(:mr) if version_component
 
-        # Supplement (e.g. ".9981sup7") - keep distinct documents distinct
-        result += supplement_short
+        # Supplement (e.g. ".9981sup7") - keep distinct documents distinct;
+        # a series-only supplement ("NBS.CIRC.sup") takes the dot itself.
+        result += if supplement
+                    number ? supplement_short : ".#{supplement_short}"
+                  else
+                    ""
+                  end
 
         # Use update_component
         result += update_component.to_s(:mr) if update_component

data/lib/pubid/nist/parser.rb

@@ -726,6 +726,15 @@ module Pubid
             # 4-digit years so it can't swallow "sup3/1926" or a base number.
             ((str("supp") | str("sup")) >> match("[0-9]").repeat(4, 4).as(:supp_year_start) >>
              dash >> match("[0-9]").repeat(4, 4).as(:supp_year_end)).as(:supplement_date_range) |
+            # Bare supplement marker to the whole series, no base number
+            # ("NBS.CIRC.sup" — testsuite#5 C4): the marker alone is the
+            # supplement.
+            ((str("supp") | str("sup")) >>
+              (
+                (month_abbrev >> digits).as(:supplement_month_year) |
+                (digits.as(:supp_number) >> slash >> digits.as(:supp_year)).as(:supplement_slash_year) |
+                str("").as(:supplement_empty)
+              ).maybe) |
             # With base identifier + supplement
             (
               # Capture base portion (everything before "supp" or "sup" or slash+year)

data/lib/pubid/nist/urn_parser.rb

@@ -42,13 +42,22 @@ module Pubid
         type_token, payload = parts
         code, revision = parse_payload(payload)
 
-        text = "NIST #{type_label(type_token)} #{code}"
+        text = "#{publisher_for(type_token)} #{type_label(type_token)} #{code}"
         text += revision if revision
         flavor_parse(text)
       end
 
       private
 
+      # The NBS-era series carry the NBS imprint in their canonical form
+      # ("NBS CSM 1") — the rebuild must name the publisher the document
+      # carries, or the flavor parse rejects its own URN's rebuild.
+      NBS_SERIES = ["csm"].freeze
+
+      def publisher_for(type_token)
+        NBS_SERIES.include?(type_token.downcase) ? "NBS" : "NIST"
+      end
+
       def parse_payload(payload)
         # Strip the trailing ".supp" supplement marker.
         stripped = payload.sub(/\.supp\z/, "")

data/lib/pubid/oasis/identifier.rb

@@ -146,7 +146,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:oasis, identifier)
         Builder.build(parsed)
       end
 

data/lib/pubid/ogc/identifier.rb

@@ -85,7 +85,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        Builder.build(Parser.parse(identifier))
+        Builder.build(Pubid::Parg::Backend.parse(:ogc, identifier))
       end
     end
   end

data/lib/pubid/oiml.rb

@@ -37,7 +37,7 @@ module Pubid
       parser = Parser.new
       builder = Builder.new
 
-      parsed = parser.parse(identifier)
+      parsed = Pubid::Parg::Backend.parse(:oiml, identifier)
       builder.build(parsed)
     end
 

data/lib/pubid/omg/identifier.rb

@@ -53,7 +53,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:omg, identifier)
         Builder.build(parsed)
       end
     end

data/lib/pubid/parg/artifact.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "parsanol"
+
+module Pubid
+  module Parg
+    # A baked, checksum-verified PG artifact for one flavor. The artifact
+    # is the parser of record for the flavor: its grammar, tests, entries,
+    # and embedded tables travel together and are verified on load.
+    class Artifact
+      DATA_DIR = File.expand_path("../../../data/parg", __dir__)
+      # The deferred entity atoms resolve from_table references at parse
+      # time from this dir, so the table YAMLs vendor alongside the
+      # baked artifacts.
+      TABLES_DIR = File.join(DATA_DIR, "tables")
+
+      @artifacts = {}
+
+      class << self
+        # Load and memoize the artifact for a flavor (e.g. :iso).
+        def for(flavor)
+          @artifacts[flavor] ||=
+            begin
+              path = File.join(DATA_DIR, "#{flavor}.json")
+              new(Parsanol::PARG::Artifact.load(path, tables_dir: TABLES_DIR))
+            rescue Parsanol::PARG::Error => e
+              raise Pubid::Errors::ParseError,
+                    "PG artifact #{path} rejected: #{e.message}"
+            end
+        end
+      end
+
+      def initialize(artifact)
+        @artifact = artifact
+      end
+
+      def parse(entry, input)
+        @artifact.parse(entry, input)
+      end
+
+      def checksum
+        @artifact.envelope["checksum"]
+      end
+    end
+  end
+end

data/lib/pubid/parg/backend.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Parg
+    # The PG-artifact parse backend: runs the flavor's baked artifact on
+    # an input and returns the builder-ready attribute hash the flavor's
+    # Builder expects — the same hash its parslet parser produced.
+    module Backend
+      LEAF_KEYS = %i[value line column offset length].freeze
+
+      module_function
+
+      ALL_PARTS_SUFFIX = "(all parts)"
+
+      def parse(flavor, input, entry: "identifier", merge_top_sequence: true)
+        # The parslet grammar base strips the "(all parts)" suffix before
+        # parsing and marks the tree (Grammar#parse / #mark_all_parts);
+        # the artifact backend carries the same contract so every flavor
+        # whose grammar does not itself consume the suffix keeps working.
+        if input.end_with?(ALL_PARTS_SUFFIX)
+          base = input.sub(/\s*\(all parts\)\s*\z/, "")
+          tree = to_builder_hash(Artifact.for(flavor).parse(entry, base), merge: merge_top_sequence)
+          return mark_all_parts(tree)
+        end
+
+        to_builder_hash(Artifact.for(flavor).parse(entry, input), merge: merge_top_sequence)
+      rescue Parsanol::ParseFailed => e
+        raise Pubid::Errors::ParseError.new(e.message, nil,
+                                            input: input,
+                                            flavor: registered_name(flavor))
+      end
+
+      # The registered flavor name (Registry-resolvable; "3gpp" for
+      # Pubid::Tgpp), falling back to the internal symbol when the
+      # flavor is not registered.
+      def registered_name(flavor)
+        mod = Pubid.const_get(flavor.to_s.split("_").map(&:capitalize).join)
+        names = Pubid::Registry.flavor_names.select { |name| Pubid::Registry.get(name) == mod }
+        # A module may register under several names; the longest is the
+        # canonical one ("cen_cenelec", not the "cen" alias).
+        names.max_by(&:length)
+      rescue NameError
+        nil
+      end || flavor.to_s
+
+      def mark_all_parts(tree)
+        case tree
+        when Hash then tree.merge(all_parts: true)
+        when Array then tree.map { |t| t.merge(all_parts: true) }
+        else tree
+        end
+      end
+
+      # The artifact emits the parsanol-tree wire shape: capture leaves
+      # ({value, line, column, offset, length}) carry their text, and the
+      # top-level sequence is a list of capture hashes. The builder-ready
+      # form scalarizes leaves and folds that top-level list into one
+      # attribute hash (last key wins) — parslet's sequence fold.
+      def to_builder_hash(shape, merge: true)
+        normalized = normalize(shape)
+        return normalized.reduce(:merge) if merge && mergeable_sequence?(normalized)
+
+        normalized
+      end
+
+      def normalize(node)
+        case node
+        when Parsanol::Slice then node.content
+        when Hash then normalize_hash(node)
+        when Array then node.map { |item| normalize(item) }
+        else node
+        end
+      end
+
+      def normalize_hash(node)
+        return node[:value] if wire_leaf?(node)
+
+        node.transform_values { |value| normalize(value) }
+      end
+
+      def mergeable_sequence?(value)
+        value.is_a?(Array) && value.all?(Hash)
+      end
+
+      def wire_leaf?(node)
+        return false unless node.size == LEAF_KEYS.size
+
+        LEAF_KEYS.all? { |key| node.key?(key) }
+      end
+    end
+  end
+end

data/lib/pubid/parg.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Parg
+    autoload :Artifact, "pubid/parg/artifact"
+    autoload :Backend, "pubid/parg/backend"
+  end
+end

data/lib/pubid/pg.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Pg
+    autoload :Artifact, "pubid/pg/artifact"
+    autoload :Backend, "pubid/pg/backend"
+  end
+end

data/lib/pubid/plateau.rb

@@ -31,8 +31,7 @@ module Pubid
 
       # Apply legacy update_codes normalization first
       normalized = Core::UpdateCodes.apply(input, :plateau)
-      parser = Parser.new
-      parsed = parser.parse(normalized)
+      parsed = Pubid::Parg::Backend.parse(:plateau, normalized)
       Builder.build(parsed)
     end
 

data/lib/pubid/sae/identifiers/base.rb

@@ -16,7 +16,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(input)
+        parsed = Pubid::Parg::Backend.parse(:sae, input)
         Builder.build(parsed)
       end
 

data/lib/pubid/tgpp/identifier.rb

@@ -113,7 +113,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:tgpp, identifier)
         Builder.build(parsed)
       end
     end

data/lib/pubid/un/identifier.rb

@@ -32,7 +32,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:un, identifier, merge_top_sequence: false)
         Builder.build(parsed)
       end
     end

data/lib/pubid/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pubid
-  VERSION = "2.0.0.pre.alpha.13"
+  VERSION = "2.0.0.pre.alpha.14"
 end

data/lib/pubid/w3c/identifier.rb

@@ -122,7 +122,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:w3c, identifier)
         Builder.build(parsed)
       end
     end

data/lib/pubid/xsf/identifier.rb

@@ -58,7 +58,7 @@ module Pubid
           raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
         end
 
-        parsed = Parser.parse(identifier)
+        parsed = Pubid::Parg::Backend.parse(:xsf, identifier)
         Builder.build(parsed)
       end
     end

data/lib/pubid.rb

@@ -17,6 +17,7 @@ require "pubid/prefixes_support"
 module Pubid
   autoload :Schema, "pubid/schema"
   autoload :Conformance, "pubid/conformance"
+  autoload :Parg, "pubid/parg"
   # Upper bound on the length of an identifier string accepted by any +parse+
   # entry point. Real-world standards identifiers are well under 200 characters;
   # this limit exists purely to keep pathological, attacker-controlled inputs