pubid 2.0.0.pre.alpha.7 → 2.0.0.pre.alpha.9

data/lib/pubid/itu/identifiers/appendix_of_recommendation.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Itu
+    module Identifiers
+      # A labelled appendix of a Recommendation — "ITU-T G.101 App. I
+      # (05/2000)", "ITU-T G.722 (1988) App. IV (11/2006)".
+      #
+      # Structurally the twin of AnnexOfRecommendation: an appendix is
+      # published and dated separately from the Recommendation it appends, and
+      # may itself carry an amendment/errata ("ITU-T G.729 App. I (2002)
+      # Amd. 1"). The label is a Roman numeral rather than a series letter.
+      class AppendixOfRecommendation < Identifier
+        attribute :base, Identifier, polymorphic: true
+        # The appendix label ("I", "II", "IV"). Named `number` for symmetry
+        # with Supplement's ordinal and AnnexOfRecommendation's label; it
+        # shadows Identifier#number (code&.number), which is nil here anyway —
+        # the document number is reached via #root.
+        attribute :number, :string
+        # The companion artefact published under the appendix's designation:
+        # "test vectors" or "Software". Its own catalogue record, so it must
+        # stay distinct from the appendix itself.
+        attribute :material, :string
+
+        # Compact serialization: the nested (itself-flat) base plus this
+        # document's own label, date and language. sector/series/code belong to
+        # the base and are deliberately not re-emitted.
+        key_value do
+          map "_type", to: :_type
+          map "number", to: :number
+          map "material", to: :material
+          map "year", with: { to: :year_to_kv, from: :year_from_kv }
+          map "month", with: { to: :month_to_kv, from: :month_from_kv }
+          map "base", with: { to: :base_to_kv, from: :base_from_kv }
+          map "language", to: :language
+          map "common_text_twin",
+              with: { to: :common_text_twin_to_kv,
+                      from: :common_text_twin_from_kv }
+        end
+
+        # "<base urn>:appendix:<label>[:<date>]" — the appendix's own date must
+        # be there, or two editions of one appendix share a URN (the base's
+        # date rides inside base_urn, so omitting this one would be asymmetric).
+        def to_urn
+          segments = [base&.to_urn || "urn:itu", "appendix", number&.downcase]
+          # Without this the test-vector record and the appendix itself would
+          # share a URN.
+          segments << material.downcase.gsub(/[^a-z0-9]/, "") if material
+          segments << urn_date_segment
+          segments.compact.join(":")
+        end
+
+        def render_base(**opts)
+          result = base ? base.render_base(**opts) : "#{publisher}-#{sector}"
+
+          result += " App. #{number}"
+          result += " #{material}" if material
+          result + render_date_suffix
+        end
+
+        # MR: recurse into the appendixed document and append
+        # "app.<label>[.year]" (the documented supplement-suffix hook), so the
+        # slug stays distinct per appendix. Without it the wrapper's own nil
+        # sector/series/code would collapse every appendix onto "itu.<year>".
+        def mr_supplement_suffix
+          label = "#{number}#{material}".downcase.gsub(/[^a-z0-9]/, "")
+
+          ["app", label.empty? ? nil : label, date&.year].compact.join(".")
+        end
+
+        def ==(other)
+          return false unless other.instance_of?(self.class)
+
+          base == other.base &&
+            number == other.number &&
+            material == other.material &&
+            date == other.date &&
+            language == other.language &&
+            common_text_twin == other.common_text_twin
+        end
+
+        private
+
+        def urn_date_segment
+          return nil unless date
+
+          date.month ? "#{date.month}/#{date.year}" : date.year.to_s
+        end
+      end
+    end
+  end
+end

data/lib/pubid/itu/identifiers/base.rb

@@ -2,17 +2,30 @@
 
 module Pubid
   module Itu
-    # Base class for all ITU identifiers. Canonical name Pubid::Itu::Identifier
-    # (Identifiers::Base is a back-compat alias).
+    # Base class for all ITU identifiers. Canonical name Pubid::Itu::Identifier.
     class Identifier < ::Pubid::Identifier
       # Parse an ITU identifier string into an identifier object.
       def self.parse(identifier)
-        parsed = Parser.parse(identifier)
+        parsed = Parser.parse(normalize_whitespace(identifier))
         Builder.build(parsed)
       rescue Parslet::ParseFailed => e
         raise "Failed to parse ITU identifier '#{identifier}': #{e.message}"
       end
 
+      # ITU's own listings occasionally carry a doubled space
+      # ("ITU-T D.271  (10/2016)"). Whitespace is never significant in an ITU
+      # identifier, so collapse runs of it rather than teaching every rule to
+      # tolerate them. Any string over the shared input-length guard is left
+      # untouched — `Pubid.parse` rejects it before this point, and the guard
+      # exists precisely to keep long inputs away from regexes.
+      def self.normalize_whitespace(identifier)
+        return identifier unless identifier.is_a?(String)
+        return identifier if identifier.length > ::Pubid::MAX_INPUT_LENGTH
+
+        identifier.gsub(/[[:space:]]+/, " ").strip
+      end
+      private_class_method :normalize_whitespace
+
       # Long-form ↔ ITU single-letter language code map. The parser produces
       # single-letter codes (E/F/S/R/A/C); API callers (e.g. metanorma-itu)
       # pass long-form (en/fr/es/ru/ar/zh). Storage is normalized to the
@@ -30,6 +43,36 @@ module Pubid
       attribute :code, Pubid::Itu::Components::Code
       attribute :date, Pubid::Components::Date
       attribute :language, :string
+      # "(V14)" of "ITU-T H.264 (V14) (08/2021)" — an ITU-T revision marker
+      # distinct from the publication date. Declared here (not on the leaves)
+      # because `render_base` lives here and the parser feeds it to both plain
+      # and combined recommendations, and to the nested base of a supplement.
+      # Safe from the number/stage redefinition landmine: ::Pubid::Identifier
+      # declares no `version` attribute, so this is a new one, not a retype.
+      attribute :version, :string
+      # The literal word "series" of "ITU-T E-100 series Suppl. 1" and
+      # "ITU-T E.1100 series Suppl. 1" — the document is the series group, not
+      # one Recommendation within it. It cannot live inside `series` itself
+      # because it also follows a *dotted* code, where folding it into the
+      # series token would corrupt the value every index row is keyed on.
+      # Named so the common case is the `false` default and stays out of
+      # `to_hash`.
+      attribute :series_word, :boolean, default: -> { false }
+      # A series-code document joins its mnemonic series to its number with a
+      # dash rather than a dot ("ITU-T EMC-5", "ITU-T SEC-QKD"). Keeping the
+      # two halves split — rather than storing "EMC-5" in `series` — is what
+      # keeps `root.number` non-empty for relaton-index. Named for the rare
+      # form, so the dotted majority stays out of `to_hash`.
+      attribute :series_dash, :boolean, default: -> { false }
+      # "ITU-T H.350 attachment (08/2003)" — the machine-readable attachment
+      # of a Recommendation, catalogued as its own record.
+      attribute :attachment, :boolean, default: -> { false }
+      # The upper bound of a Recommendation range published as one document —
+      # "ITU-T Q.120-Q.139 (11/1988)", where this holds "Q.139". A plain string
+      # rather than a structured designation: five legacy records, and the
+      # printed form is the whole of their identity.
+      attribute :range_end, :string
+      attribute :common_text_twin, ::Pubid::Identifier
 
       def initialize(**kwargs)
         if kwargs[:language]
@@ -41,10 +84,91 @@ module Pubid
         validate_ob_no_sector!
       end
 
+      # The document number lives on the `code` component for ITU; surface it at
+      # the identifier root so consumers that key on `#number` — e.g.
+      # Relaton::Index's sort/bsearch narrowing on `id.root.number.to_s` — work
+      # without special-casing ITU. Returns the `code.number` string (e.g.
+      # "530"); serialization is unaffected because the flat key_value block
+      # maps "number" via `number_to_kv`/`number_from_kv`, which read/write
+      # `code.number` directly and never touch the inherited `number` attribute.
+      # Supplement/Amendment/Corrigendum/Errata declare their own `:string`
+      # `number` (the ordinal), which overrides this reader; their base document
+      # number is still reachable via `root.number`.
+      def number
+        code&.number
+      end
+
+      # ITU keeps the document number and its edition/part together inside one
+      # Code component (like ETSI, unlike ISO's separate `part` attribute), so a
+      # top-level exclude(:part) can't reach the part. Handle it during the
+      # nested-exclusion pass: when :part/:parts is excluded, return a copy of
+      # the Code with its parts cleared while keeping number/series/subseries.
+      # This lets a part-less reference (e.g. `ITU-R P.838`) match every edition
+      # (`ITU-R P.838-3`) via matches?(other, ignore: [:parts]). Any non-Code
+      # value falls through to super, so nested identifiers (common_text_twin,
+      # supplement `base`, combined `Designation`s) still recurse.
+      def exclude_from_nested(value, args)
+        part_keys = args & %i[part parts]
+        if value.is_a?(Pubid::Itu::Components::Code) && !part_keys.empty?
+          return Pubid::Itu::Components::Code.new(
+            imp_marker: value.imp_marker,
+            number: value.number,
+            series_suffix: value.series_suffix,
+            series_suffix_spaced: value.series_suffix_spaced,
+            subseries: value.subseries,
+            parts: [],
+            qualifier: value.qualifier,
+            qualifier_glued: value.qualifier_glued,
+          )
+        end
+
+        super
+      end
+
       # Generate URN for this identifier
       #
       # @return [String] URN representation
 
+      # ITU encodes identity across `sector` (T/R/CD), `series` (G/H/J…), and
+      # `code` (number with optional part), not in the inherited `number`/
+      # `typed_stage`. The generic MrString renderer would otherwise drop all
+      # three and emit just `ITU`. Losslessness for issue #142 requires the
+      # sector letter, series letter, and document number to appear in MR.
+      # Lowercased to match the all-lowercase MR convention.
+      def mr_publisher
+        publisher&.to_s&.downcase
+      end
+
+      def mr_type
+        sector&.sector&.to_s&.downcase
+      end
+
+      def mr_number_with_part
+        segments = []
+        segments << series&.series&.to_s&.downcase if series&.series
+        # The edition word and the qualifier letter are part of the document's
+        # identity, so they must reach the slug: without them "D.200" and
+        # "D.200 R" — and, worse, "Q.2931 B" and "Q.2931 C" — collapse onto
+        # one MR string. They are glued to the number rather than given their
+        # own "-" segment so they read as part of it ("x-50bis", "d-200r").
+        number = code&.number&.to_s
+        if number
+          number += code.series_suffix.to_s if code.series_suffix
+          number += code.qualifier.to_s.downcase if code.qualifier
+          segments << number
+        end
+        segments << code&.subseries&.to_s if code&.subseries
+        segments.concat(code&.parts&.map(&:to_s) || [])
+        # Same reasoning as the suffixes above, and as the URN generator: these
+        # three distinguish documents that otherwise slug identically.
+        segments << "series" if series_word
+        segments << "attachment" if attachment
+        segments << "to-#{range_end.to_s.downcase.tr('.', '-')}" if range_end
+        return nil if segments.empty?
+
+        segments.join("-")
+      end
+
       # Stored as a plain string (always "ITU") so it round-trips through
       # to_hash/from_hash. Was a `def publisher` method, which made lutaml
       # serialize a String against the Components::Publisher attribute.
@@ -61,29 +185,43 @@ module Pubid
       #   identifiers, otherwise the default short form.
       def to_s(**opts)
         opts = self.class.normalize_to_s_opts(opts)
-        render_base(**opts) + render_language_suffix
+        result = render_base(**opts) + render_language_suffix
+        result += " | #{common_text_twin}" if common_text_twin
+        result
       end
 
       def render_base(**_opts)
         result = "#{publisher}-#{sector}"
 
-        # Add series and code
-        result += if series
-                    " #{series}.#{code}"
+        # Add series and code. A series with no code is a series GROUP
+        # ("ITU-T G-100 series"); without this branch it rendered a dangling
+        # trailing dot.
+        result += if series && code
+                    " #{series}#{series_dash ? '-' : '.'}#{code}"
+                  elsif series
+                    " #{series}"
                   else
                     " #{code}"
                   end
 
-        # Add date if present
-        if date
-          result += if date.month
-                      " (#{date.month}/#{date.year})"
-                    else
-                      " (#{date.year})"
-                    end
-        end
+        result += "-#{range_end}" if range_end
+        result += " series" if series_word
+        result += " attachment" if attachment
 
-        result
+        # Add version marker if present — always between code and date
+        result += " (V#{version})" if version
+
+        result + render_date_suffix
+      end
+
+      # " (MM/YYYY)", or " (YYYY)" when the month is unknown; "" when undated.
+      # Shared by every type that prints its date this way (Supplement and its
+      # subclasses, CombinedIdentifier, AnnexOfRecommendation). NOT used by
+      # SpecialPublication, which zero-pads its month.
+      def render_date_suffix
+        return "" unless date
+
+        date.month ? " (#{date.month}/#{date.year})" : " (#{date.year})"
       end
 
       def render_language_suffix
@@ -109,14 +247,199 @@ module Pubid
         opts
       end
 
+      # Class-strict, like Supplement#== — the ITU *type* is part of the
+      # identity, not just the sector/series/number. Report ITU-R BT.2020-1 and
+      # Recommendation ITU-R BT.2020-1 are two different, both-current
+      # documents; an `is_a?` guard would make them equal (and `#matches?`
+      # resolve one to the other). Using `self.class` rather than a hard-coded
+      # class keeps the comparison symmetric in both directions, and closes the
+      # same one-way holes that existed against CombinedIdentifier and
+      # SpecialPublication.
       def ==(other)
-        return false unless other.is_a?(Pubid::Itu::Identifier)
+        return false unless other.instance_of?(self.class)
 
         sector == other.sector &&
           series == other.series &&
           code == other.code &&
           date == other.date &&
-          language == other.language
+          version == other.version &&
+          # "the E.1100 series" is not Recommendation E.1100.
+          series_word == other.series_word &&
+          series_dash == other.series_dash &&
+          attachment == other.attachment &&
+          range_end == other.range_end &&
+          language == other.language &&
+          common_text_twin == other.common_text_twin
+      end
+
+      # --- Compact flat key_value converters (shared) --------------------
+      # Collapse the identity components (Sector/Series/Code/Date) to bare
+      # top-level scalars so the serialized hash is flat and index-friendly,
+      # mirroring ISO/ETSI/JCGM/OIML. Used by the per-leaf key_value blocks
+      # (see StandardSerialization) and by the Supplement/Annex/Combined
+      # blocks. `publisher` ("ITU") is intentionally never mapped — it is
+      # reconstructed from its attribute default.
+
+      def sector_to_kv(model, doc)
+        emit_kv(doc, "sector", model.sector&.sector)
+      end
+
+      def sector_from_kv(model, value)
+        model.sector = Components::Sector.new(sector: value.to_s)
+      end
+
+      def series_to_kv(model, doc)
+        emit_kv(doc, "series", model.series&.series)
+      end
+
+      def series_from_kv(model, value)
+        model.series = Components::Series.new(series: value.to_s)
+      end
+
+      def imp_marker_to_kv(model, doc)
+        emit_kv(doc, "imp_marker", model.code&.imp_marker)
+      end
+
+      def imp_marker_from_kv(model, value)
+        code_for(model).imp_marker = value.to_s
+      end
+
+      def number_to_kv(model, doc)
+        emit_kv(doc, "number", model.code&.number)
+      end
+
+      def number_from_kv(model, value)
+        code_for(model).number = value.to_s
+      end
+
+      def series_suffix_to_kv(model, doc)
+        emit_kv(doc, "series_suffix", model.code&.series_suffix)
+      end
+
+      def series_suffix_from_kv(model, value)
+        code_for(model).series_suffix = value.to_s
+      end
+
+      # The three spelling/qualifier fields of Components::Code. The two
+      # booleans emit only when true, so no already-published index row gains
+      # a key; `qualifier` emits only when present.
+      def series_suffix_spaced_to_kv(model, doc)
+        return unless model.code&.series_suffix_spaced
+
+        doc["series_suffix_spaced"] = true
+      end
+
+      def series_suffix_spaced_from_kv(model, value)
+        code_for(model).series_suffix_spaced = value
+      end
+
+      def qualifier_to_kv(model, doc)
+        emit_kv(doc, "qualifier", model.code&.qualifier)
+      end
+
+      def qualifier_from_kv(model, value)
+        code_for(model).qualifier = value.to_s
+      end
+
+      def qualifier_glued_to_kv(model, doc)
+        return unless model.code&.qualifier_glued
+
+        doc["qualifier_glued"] = true
+      end
+
+      def qualifier_glued_from_kv(model, value)
+        code_for(model).qualifier_glued = value
+      end
+
+      def subseries_to_kv(model, doc)
+        emit_kv(doc, "subseries", model.code&.subseries)
+      end
+
+      def subseries_from_kv(model, value)
+        code_for(model).subseries = value.to_s
+      end
+
+      def parts_to_kv(model, doc)
+        parts = model.code&.parts
+        return if parts.nil? || parts.empty?
+
+        doc.add_child(
+          Lutaml::KeyValue::DataModel::Element.new("parts", parts.map(&:to_s)),
+        )
+      end
+
+      def parts_from_kv(model, value)
+        code_for(model).parts = Array(value).map(&:to_s)
+      end
+
+      def year_to_kv(model, doc)
+        emit_kv(doc, "year", model.date&.year)
+      end
+
+      def year_from_kv(model, value)
+        date_for(model).year = value.to_s
+      end
+
+      def month_to_kv(model, doc)
+        emit_kv(doc, "month", model.date&.month)
+      end
+
+      def month_from_kv(model, value)
+        date_for(model).month = value.to_s
+      end
+
+      # Serialize a nested `base` identifier via its own to_hash so it collapses
+      # to the compact shape, and re-dispatch through the flavor base on load so
+      # `_type` resolves to the concrete subclass (a bare polymorphic cast would
+      # rebuild a plain Identifier and lose the subclass). Mirrors ETSI/JCGM.
+      def base_to_kv(model, doc)
+        return unless model.base
+
+        doc.add_child(
+          Lutaml::KeyValue::DataModel::Element.new("base", model.base.to_hash),
+        )
+      end
+
+      def base_from_kv(model, value)
+        return unless value
+
+        model.base = Pubid::Itu::Identifier.from_hash(value)
+      end
+
+      # The common-text twin ("| ISO/IEC 13818-1:2022") is another flavor's
+      # identifier (ISO/IEC …); serialize it via its own flat to_hash and
+      # reconstruct it cross-flavor through the top-level polymorphic router so
+      # its concrete class (and custom mappings) are restored by `_type`.
+      def common_text_twin_to_kv(model, doc)
+        return unless model.common_text_twin
+
+        doc.add_child(
+          Lutaml::KeyValue::DataModel::Element.new(
+            "common_text_twin", model.common_text_twin.to_hash
+          ),
+        )
+      end
+
+      def common_text_twin_from_kv(model, value)
+        return unless value
+
+        model.common_text_twin = ::Pubid::Identifier.from_hash(value)
+      end
+
+      def emit_kv(doc, key, value)
+        return if value.nil? || value.to_s.empty?
+
+        doc.add_child(
+          Lutaml::KeyValue::DataModel::Element.new(key, value.to_s),
+        )
+      end
+
+      def code_for(model)
+        model.code ||= Components::Code.new
+      end
+
+      def date_for(model)
+        model.date ||= Pubid::Components::Date.new
       end
 
       private
@@ -141,10 +464,5 @@ module Pubid
       end
     end
 
-    module Identifiers
-      # Backward-compatible alias: ITU's base class used to be
-      # Pubid::Itu::Identifiers::Base. It is now Pubid::Itu::Identifier.
-      Base = Pubid::Itu::Identifier
-    end
   end
 end

data/lib/pubid/itu/identifiers/combined_identifier.rb

@@ -3,41 +3,117 @@
 module Pubid
   module Itu
     module Identifiers
-      # Combined identifier for dual-series recommendations
-      # Format: ITU-T G.780/Y.1351
-      # Example: ITU-T G.780/Y.1351 (2004)
-      class CombinedIdentifier < Base
-        attribute :combined_series, Pubid::Itu::Components::Series
-        attribute :combined_code, Pubid::Itu::Components::Code
-
-        def to_s
+      # Combined (joint) recommendation — a single document published under two
+      # or more series/number designations at once.
+      # Format: ITU-T G.780/Y.1351         (dual)
+      #         ITU-T G.780/Y.1351/Z.1362  (triple)
+      #
+      # The primary designation lives on the base `series`/`code`; every
+      # additional designation is a Components::Designation in `combined`
+      # (one for a dual form, two+ for triple).
+      class CombinedIdentifier < Identifier
+        include StandardSerialization
+
+        attribute :combined, Pubid::Itu::Components::Designation,
+                  collection: true, default: -> { [] }
+
+        # Extra map merged on top of StandardSerialization: the additional
+        # designations as a flat list of { series, number, subseries?, parts? }.
+        key_value do
+          map "combined",
+              with: { to: :combined_to_kv, from: :combined_from_kv }
+        end
+
+        def combined_to_kv(model, doc)
+          designations = model.combined
+          return if designations.nil? || designations.empty?
+
+          rows = designations.map do |d|
+            row = { "series" => d.series&.series.to_s,
+                    "number" => d.code&.number.to_s }
+            row["subseries"] = d.code.subseries.to_s if d.code&.subseries
+            row["parts"] = d.code.parts.map(&:to_s) if d.code&.parts&.any?
+            # Each emitted only when set, so an ordinary joint designation's
+            # row keeps its existing two-key shape.
+            if d.code&.series_suffix
+              row["series_suffix"] = d.code.series_suffix.to_s
+              row["series_suffix_spaced"] = true if d.code.series_suffix_spaced
+            end
+            if d.code&.qualifier
+              row["qualifier"] = d.code.qualifier.to_s
+              row["qualifier_glued"] = true if d.code.qualifier_glued
+            end
+            row
+          end
+
+          doc.add_child(
+            Lutaml::KeyValue::DataModel::Element.new("combined", rows),
+          )
+        end
+
+        def combined_from_kv(model, value)
+          model.combined = Array(value).map do |row|
+            row = row.transform_keys(&:to_s)
+            Components::Designation.new(
+              series: Components::Series.new(series: row["series"].to_s),
+              code: Components::Code.new(
+                number: row["number"].to_s,
+                series_suffix: row["series_suffix"]&.to_s,
+                series_suffix_spaced: !!row["series_suffix_spaced"],
+                subseries: row["subseries"]&.to_s,
+                parts: Array(row["parts"]).map(&:to_s),
+                qualifier: row["qualifier"]&.to_s,
+                qualifier_glued: !!row["qualifier_glued"],
+              ),
+            )
+          end
+        end
+
+        # Rendering hooks into `render_base` (not `to_s`) so that any wrapper
+        # composing this identifier — AnnexOfRecommendation, which renders
+        # "<base.render_base> Annex <label>" — keeps the co-designations.
+        # Overriding `to_s` instead silently dropped the "/Y.1351" half.
+        # The inherited `to_s` adds the language suffix and common-text twin.
+        def render_base(**_opts)
           result = "#{publisher}-#{sector}"
 
           # Add primary series and code
           result += if series
-                      " #{series}.#{code}"
+                      " #{series}#{series_dash ? '-' : '.'}#{code}"
                     else
                       " #{code}"
                     end
 
-          # Add combined series and code
-          if combined_series && combined_code
-            result += "/#{combined_series}.#{combined_code}"
-          end
+          result += "-#{range_end}" if range_end
 
-          # Add date if present
-          if date
-            result += if date.month
-                        " (#{date.month}/#{date.year})"
-                      else
-                        " (#{date.year})"
-                      end
+          # Add additional designations
+          if combined&.any?
+            result += "/#{combined.join('/')}"
           end
 
-          # Add language
-          result += "-#{language}" if language
+          result += " series" if series_word
+          result += " attachment" if attachment
+
+          # Add version marker if present — always between code and date
+          result += " (V#{version})" if version
+
+          result + render_date_suffix
+        end
+
+        def ==(other)
+          return false unless other.is_a?(CombinedIdentifier)
 
-          result
+          sector == other.sector &&
+            series == other.series &&
+            code == other.code &&
+            combined == other.combined &&
+            date == other.date &&
+            version == other.version &&
+            series_word == other.series_word &&
+            series_dash == other.series_dash &&
+            attachment == other.attachment &&
+            range_end == other.range_end &&
+            language == other.language
         end
       end
     end