pubid 2.0.0.pre.alpha.8 → 2.0.0.pre.alpha.11

data/lib/pubid/cie/single_identifier.rb

@@ -24,7 +24,46 @@ module Pubid
       attribute :publisher, :string, default: -> { "CIE" }
 
       attribute :year, :string
-      attribute :date_separator, :string # "dash" or "colon"
+      # The number<->year separator character, derived from +style+ (the sole
+      # separator field; there is no date_separator attribute):
+      #   current -> ":"   legacy -> "-"   slash -> "/"
+      def date_sep_char
+        { "legacy" => "-", "slash" => "/" }.fetch(style, ":")
+      end
+
+      # Partial-reference matching: CIE keeps the publication year in its own
+      # `year` string attribute (the separator lives in `style`), not a `Date`
+      # component, so the base's `:year`->`:date` remap would nil the unused
+      # inherited `date` and leave `year` intact. Call super (preserving the
+      # nested-identifier recursion), then nil `year` directly so a year-less
+      # CIE ref is a year wildcard under `matches?(ignore: [:year])`.
+      #
+      # Known limit: `style` (which also encodes the era: current<->colon,
+      # legacy<->dash) is intentionally NOT wildcarded, so bare `CIE 015`
+      # (always current) matches a colon-dated full but not a legacy dash-dated
+      # (pre-2001) one. Partial-ref scope targets modern colon forms.
+      def exclude(*args)
+        result = super
+        result.year = nil if args.include?(:year) || args.include?(:date)
+        result
+      end
+
+      # CIE stores its publication year in its own `year` string attribute
+      # (not a Components::Date), and uses a single-Language component with
+      # multiple formats. Override the lossless MR template so `CIE S 017/E:2011`
+      # round-trips as `CIE.S.017.E.2011` rather than losing the type letter,
+      # language, and year (issue #142). Subclasses add the `s_prefix` and
+      # `language` attributes that this template reads.
+      def mr_year
+        year&.to_s
+      end
+
+      def mr_languages
+        return nil unless respond_to?(:language) && language
+
+        # Lowercase + filename-safe: drop the leading `/` (path-unsafe).
+        language.code.to_s.downcase
+      end
     end
   end
 end

data/lib/pubid/cie/supplement_identifier.rb

@@ -2,25 +2,38 @@
 
 module Pubid
   module Cie
-    # Base class for CIE supplement identifiers
-    # Single Responsibility: Provide common attributes for supplement documents
+    # Base class for CIE supplement identifiers (Supplement, Corrigendum).
     #
-    # Supplement identifiers modify or extend base identifiers.
-    # Unlike ISO/IEC, CIE supplements store base information as flat
-    # string attributes rather than recursive Identifier objects.
+    # Like ISO/IEC/BSI/JCGM, a CIE supplement wraps its base document as a
+    # nested full identifier (+base+, a Pubid::Cie::Identifier),
+    # rather than flat base_number/base_year strings. This lets CIE supplements
+    # participate in the shared base_document / drop_supplements / matches?
+    # vocabulary relaton uses to match a citation to its base standard.
     #
-    # Classes inheriting from SupplementIdentifier:
-    # - Corrigendum (corrigenda with /CorN notation)
-    # - Supplement (supplements with -SPN notation)
-    #
-    # Architecture Note:
-    # CIE uses a different architectural approach than ISO/IEC:
-    # - ISO/IEC: Recursive base_identifier objects
-    # - CIE: Flat string attributes (base_number, base_year, etc.)
+    # Concrete subtypes: Supplement (-SPN), Corrigendum (/CorN).
     class SupplementIdentifier < Identifier
-      # Common attribute for all CIE identifiers
-      # Values: "current" (colon date separator) or "legacy" (dash date separator)
-      attribute :style, :string
+      attribute :base, Identifier, polymorphic: true
+      # The supplement's/corrigendum's own ordinal ("1" in -SP1 / Cor1).
+      # :string overrides the base ::Pubid::Identifier Components::Code type.
+      # (Not the index key — relaton keys on #root.number, the base standard's.)
+      attribute :number, :string
+
+      # Uniform, class-agnostic supplement interface (shared by Supplement and
+      # Corrigendum) so callers never special-case corrigenda. +supplement_type+
+      # and +supplement_year+ are defined per subclass.
+      def supplement_number
+        number
+      end
+
+      # The underlying standard, with supplement layer(s) peeled recursively.
+      def base_document
+        base&.base_document || self
+      end
+
+      # Dropping one supplement layer yields the wrapped base identifier.
+      def drop_supplements
+        base || self
+      end
     end
   end
 end

data/lib/pubid/cie/urn_generator.rb

@@ -15,8 +15,8 @@ module Pubid
 
         parts << publisher_component
 
-        if identifier.code
-          parts << identifier.code.to_s
+        if identifier.respond_to?(:code_string) && identifier.code_string
+          parts << identifier.code_string
         end
 
         year = extract_year
@@ -37,8 +37,8 @@ module Pubid
         stage = maybe(:stage)
         parts << stage.to_s.downcase if stage
 
-        date_separator = maybe(:date_separator)
-        parts << "sep.#{date_separator}" if date_separator
+        # style is the sole separator field; record it for dated identifiers
+        parts << "sep.#{maybe(:style)}" if year
 
         iec_identifier = maybe(:iec_identifier)
         parts << "iec.#{iec_identifier}" if iec_identifier
@@ -49,11 +49,14 @@ module Pubid
         doc_type = maybe(:doc_type)
         parts << "doctype.#{doc_type}" if doc_type
 
-        identifiers_string = maybe(:identifiers_string)
-        parts << "bundle.#{identifiers_string}" if identifiers_string
+        if identifier.respond_to?(:ids) && identifier.ids&.any?
+          parts << "bundle.#{identifier}"
+        end
 
-        bundle_number = maybe(:bundle_number)
-        parts << "tut-bundle.#{bundle_number}" if bundle_number
+        if identifier.is_a?(Cie::Identifiers::TutorialBundle) &&
+           identifier.number
+          parts << "tut-bundle.#{identifier.number}"
+        end
 
         languages = maybe(:languages)
         if languages&.any?

data/lib/pubid/cie.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "pubid"
 module Pubid
   module Cie
     extend Pubid::PrefixesSupport
@@ -8,7 +9,6 @@ module Pubid
     PREFIXES = ["CIE"].freeze
 
     module Components
-      autoload :Code, "#{__dir__}/cie/components/code"
       autoload :Language, "#{__dir__}/cie/components/language"
     end
 
@@ -24,8 +24,13 @@ module Pubid
     # Main entry point for CIE identifiers
     # Delegates to Identifier.parse
     def self.parse(input)
+      unless input.is_a?(String)
+        raise Pubid::Errors::InvalidInputError,
+              Pubid::INPUT_NOT_A_STRING_MESSAGE
+      end
+
       if input.length > Pubid::MAX_INPUT_LENGTH
-        raise ArgumentError, Pubid::INPUT_TOO_LONG_MESSAGE
+        raise Pubid::Errors::InvalidInputError, Pubid::INPUT_TOO_LONG_MESSAGE
       end
 
       Identifier.parse(input)

data/lib/pubid/components/date.rb

@@ -6,19 +6,33 @@ module Pubid
   module Components
     # Publication date component
     #
-    # Human render: "YYYY" or "YYYY-MM" or "YYYY-MM-DD".
-    # URN render: year only (RFC 5141-bis URN spec).
+    # Human render: "YYYY" or "YYYY-MM" or "YYYY-MM-DD"; or "--" when the
+    # date slot is present but explicitly undated (ISO/IEC undated-reference
+    # convention, e.g. "ISO 16634:--"). URN render: year only (RFC 5141-bis
+    # URN spec); flavors that drop the slot when undated gate that decision
+    # at the URN-generator level, not here.
     class Date < Lutaml::Model::Serializable
       attribute :year, :string
       attribute :month, :string
       attribute :day, :string
+      attribute :undated, :boolean, default: false
 
-      # True when the year is set to a non-empty value.
+      # True when the year is set to a non-empty value, OR the date is
+      # explicitly marked undated. An undated date is a meaningful
+      # publication-date slot (it is what renders as `:--` in ISO/IEC), so
+      # callers that gate rendering on "is there a date at all?" should
+      # treat undated as present.
       def present?
+        return true if undated
         !year.nil? && !year.to_s.empty?
       end
 
+      def undated?
+        !!undated
+      end
+
       def render(context: nil)
+        return "--" if undated? && (year.nil? || year.to_s.empty?)
         return nil unless present?
         return year.to_s if context&.urn?
         return year.to_s unless month
@@ -29,6 +43,7 @@ module Pubid
       end
 
       def to_s(_context: nil, _include_month: false)
+        return "--" if undated? && (year.nil? || year.to_s.empty?)
         return year.to_s unless month
 
         result = "#{year}-#{pad2(month)}"
@@ -43,13 +58,14 @@ module Pubid
       end
 
       def hash
-        @hash ||= [year, month, day].compact.map(&:hash).hash
+        @hash ||= [year, month, day, undated].compact.map(&:hash).hash
       end
 
       def eql?(other)
         return false unless other.is_a?(self.class)
 
-        year == other.year && month == other.month && day == other.day
+        year == other.year && month == other.month && day == other.day &&
+          undated == other.undated
       end
     end
   end

data/lib/pubid/conformance/checks.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # The single check core shared by the conformance runner and the rspec
+    # corpus gate. One corpus case in, its mismatch list out (empty ==
+    # passing). The list shape is load-bearing: RSpec's aggregate_failures
+    # does not raise on a failed expectation, so exception-based control
+    # flow would misread real failures as passes - and the runner needs to
+    # accumulate across cases rather than abort.
+    module Checks
+      class << self
+        # @param test_case [Pubid::Conformance::Corpus::Case]
+        # @param flavor_module [Module] the flavor owning the case
+        # @return [Array<String>] mismatch descriptions, empty when passing
+        def check_case(test_case, flavor_module)
+          identifier = flavor_module.parse(test_case.representations.human)
+          mismatchers(identifier, test_case, flavor_module)
+            .flat_map(&:call)
+        rescue StandardError => e
+          ["#{test_case.id} raised #{e.class}"]
+        end
+
+        # Each checker returns the case's mismatch list for its concern.
+        def mismatchers(identifier, test_case, flavor_module)
+          label = test_case.id.to_s
+          [
+            -> { canonical_mismatches(identifier, test_case, label) },
+            -> { representation_mismatches(identifier, test_case, label) },
+            -> { alias_mismatches(test_case, flavor_module, label) },
+            -> { deserialize_mismatches(identifier, flavor_module, label) },
+          ]
+        end
+
+        def canonical_mismatches(identifier, test_case, label)
+          actual = Conformance.plainify(identifier.to_hash)
+          actual == test_case.identifier ? [] : ["#{label} canonical hash"]
+        end
+
+        private
+
+        def representation_mismatches(identifier, test_case, label)
+          mismatches = []
+          reps = test_case.representations
+          mismatches << "#{label} human" if identifier.to_s != reps.human
+          if reps.urn && identifier.to_urn != reps.urn
+            mismatches << "#{label} urn"
+          end
+          mismatches
+        end
+
+        def alias_mismatches(test_case, flavor_module, label)
+          human = test_case.representations.human
+          test_case.non_normalized_aliases.filter_map do |entry|
+            aliased = flavor_module.parse(entry.spelling)
+            "#{label} alias #{entry.spelling}" if aliased.to_s != human
+          rescue StandardError
+            "#{label} alias #{entry.spelling} raised"
+          end
+        end
+
+        def deserialize_mismatches(identifier, flavor_module, label)
+          hash = identifier.to_hash
+          ok = flavor_module::Identifier.from_hash(hash).to_hash == hash
+          ok ? [] : ["#{label} deserialize"]
+        rescue StandardError
+          ["#{label} deserialize raised"]
+        end
+      end
+    end
+  end
+end

data/lib/pubid/conformance/corpus/case.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    module Corpus
+      # Every proper rendering of the case's identifier.
+      class Representations < Lutaml::Model::Serializable
+        attribute :human, :string
+        attribute :urn, :string
+      end
+
+      # One old spelling that must normalize to the human form.
+      class Spelling < Lutaml::Model::Serializable
+        attribute :spelling, :string
+        attribute :style, :string
+      end
+
+      # Neutral rejection expectation - never an implementation class.
+      class Error < Lutaml::Model::Serializable
+        attribute :code, :string
+      end
+
+      class Expectation < Lutaml::Model::Serializable
+        attribute :error, Error
+      end
+
+      # One corpus case (schema/test.schema.yaml is the wire contract).
+      # `identifier` stays an opaque hash on purpose: it is the flavor's
+      # own canonical to_hash carried as DATA to compare against, not a
+      # structure the reader interprets.
+      class Case < Lutaml::Model::Serializable
+        attribute :id, :string
+        attribute :style, :string
+        attribute :identifier, :hash
+        attribute :representations, Representations
+        attribute :non_normalized_aliases, Spelling, collection: true,
+                                                     default: -> { [] }
+        attribute :roundtrip, :boolean
+        attribute :expect, Expectation
+        attribute :input, :string
+        attribute :notes, :string
+        attribute :review, :string
+
+        def error_case? = !expect.nil? && !expect.error.nil?
+
+        # Reference-bug debt: nothing to gate on.
+        def quarantined? = identifier.nil? && !error_case?
+
+        def review? = !review.to_s.empty?
+
+        def roundtrip_failure_expected? = roundtrip == false
+      end
+    end
+  end
+end

data/lib/pubid/conformance/corpus.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Typed reading layer for pubid-testsuite payloads. The YAML schema
+    # (schema/test.schema.yaml in pubid-testsuite) is the wire contract;
+    # these models are the shared typed reader for every implementation -
+    # the Ruby gem now, pubid-ts mirroring the same shape later. No
+    # consumer pokes payload hashes directly.
+    module Corpus
+      autoload :Case, "pubid/conformance/corpus/case"
+
+      class << self
+        # Cases from one payload file (underscore payloads excluded).
+        def load_file(path)
+          Array(YAML.safe_load_file(path)).map { |doc| Case.from_hash(doc) }
+        end
+
+        def case_files(flavor, corpus_dir)
+          Dir[File.join(corpus_dir, flavor, "*.yaml")]
+            .reject { |p| File.basename(p).start_with?("_") }.sort
+        end
+
+        def cases(flavor, corpus_dir)
+          case_files(flavor, corpus_dir).flat_map { |p| load_file(p) }
+        end
+
+        # The must-fail payload (fail fixtures decoded by the exporter).
+        def negative_file(flavor, corpus_dir)
+          path = File.join(corpus_dir, flavor, "_negative.yaml")
+          File.exist?(path) ? [path] : []
+        end
+      end
+    end
+  end
+end

data/lib/pubid/conformance/generator.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Migrates the ground-truth fixture library (spec/fixtures/{flavor}/
+    # identifiers/{pass,fail}/*.txt) into the neutral corpus by running the
+    # reference implementation. Fixture lines that normalize to the same
+    # identifier collapse into one canonical case carrying the old
+    # spellings as non_normalized_aliases (old => normalized, and the
+    # normalized form round-trips). Unparseable pass lines are recorded as
+    # visible debt; fail lines (stored as "#ID# Error: ..." comments in the
+    # fixture files) become negative cases. Nothing is sampled, filtered,
+    # or dropped.
+    class Generator
+      attr_reader :flavor
+
+      def initialize(flavor)
+        require "pubid"
+
+        @flavor = flavor.to_s
+        Pubid.eager_load_flavors!
+        @flavor_module = Pubid::Registry.get(@flavor)
+        raise ArgumentError, "unknown flavor #{@flavor}" unless @flavor_module
+      end
+
+      def generate(output_dir:)
+        results = { files: 0, cases: 0, negative: 0, debt: 0, aliases: 0,
+                    duplicates: 0, reclassify: 0, roundtrip_failures: 0 }
+        debt = []
+        pass_files.each do |path|
+          type = File.basename(path, ".txt")
+          cases, type_debt = build_pass_cases(path, type, results)
+          debt.concat(type_debt)
+          next if cases.empty?
+
+          write_yaml(File.join(output_dir, "#{type}.yaml"), cases)
+          results[:files] += 1
+        end
+        results[:debt] = debt.size
+        write_yaml(File.join(output_dir, "_debt.yaml"), debt) if debt.any?
+        negative = build_negative_cases(results)
+        if negative.any?
+          write_yaml(File.join(output_dir, "_negative.yaml"), negative)
+        end
+        results
+      end
+
+      private
+
+      def pass_files
+        Dir[File.join(fixtures_dir, "identifiers", "pass", "*.txt")] +
+          Dir[File.join(fixtures_dir, "identifiers", "full", "*.txt")]
+      end
+
+      def fail_files
+        Dir[File.join(fixtures_dir, "identifiers", "fail", "*.txt")]
+      end
+
+      def fixtures_dir
+        File.join(tests_repo, "reference-docs", @flavor)
+      end
+
+      def tests_repo
+        @tests_repo ||= ENV.fetch("PUBID_TESTSUITE_PATH",
+                                  File.expand_path("../../../../pubid-testsuite",
+                                                   __dir__))
+      end
+
+      def fixture_lines(path)
+        File.readlines(path).map(&:strip).reject do |line|
+          line.empty? || line.start_with?("#")
+        end
+      end
+
+      # Fail fixtures store identifiers as comment lines shaped
+      # "#IDENTIFIER# Parslet::ParseFailed: ...". Strip the leading # and
+      # the trailing historical annotation to recover the input.
+      def fail_fixture_lines(path)
+        File.readlines(path).filter_map do |raw|
+          line = raw.strip
+          next if line.empty?
+          next line unless line.start_with?("#")
+
+          input = line.sub(/\A#/, "").split("#", 2).first.to_s.strip
+          input unless input.empty?
+        end
+      end
+
+      def build_pass_cases(path, type, results)
+        groups = {}
+        debt = []
+        fixture_lines(path).each do |input|
+          record = build_record(input)
+          if record.key?(:debt)
+            debt << record[:debt]
+            next
+          end
+          key = JSON.generate(record[:case]["identifier"]) +
+            record[:case]["representations"]["human"]
+          (groups[key] ||= { record: record[:case], 
+                             inputs: [] })[:inputs] << input
+        end
+        cases = assign_ids(groups, type, results)
+        [cases, debt]
+      end
+
+      def assign_ids(groups, type, results)
+        groups.values.sort_by { |g| canonical_input(g) }.each_with_index
+          .map do |group, index|
+          record = group[:record]
+          record["id"] = format("%s.%s.%04d", @flavor, type, index + 1)
+          canonical = canonical_input(group)
+          record["style"] = Conformance.style_for(canonical)
+          inputs = group[:inputs]
+          duplicates = [inputs.count(canonical) - 1, 0].max
+          results[:duplicates] += duplicates
+          aliases = inputs.reject { |i| i == canonical }.uniq.sort
+          results[:aliases] += aliases.size
+          unless aliases.empty?
+            record["non_normalized_aliases"] = aliases.map do |a|
+              { "input" => a, "style" => Conformance.style_for(a) }
+            end
+          end
+          results[:roundtrip_failures] += 1 if record["roundtrip"] == false
+          results[:cases] += 1
+          record
+        end
+      end
+
+      def canonical_input(group)
+        group.dig(:record, "representations", "human")
+      end
+
+      def build_record(input)
+        identifier = @flavor_module.parse(input)
+        hash = identifier.to_hash
+        representations = { "human" => identifier.to_s }
+        urn = safe_urn(identifier)
+        representations["urn"] = urn if urn
+        record = { "id" => nil,
+                   "identifier" => Conformance.plainify(hash) }
+        record["representations"] = representations
+        record["roundtrip"] = false unless round_trips?(hash)
+        quarantine!(record) if bundled_month_bug?(hash)
+        { case: record }
+      rescue StandardError => e
+        { debt: { "id" => "#{@flavor}.debt.#{input.hash.abs}",
+                  "input" => input,
+                  "expect" => { "error" =>
+                                { "code" => Conformance.error_code_for(e) } },
+                  "notes" => "unparsed ground-truth fixture: " \
+                             "#{e.class}: #{e.message.to_s[0, 200]}" } }
+      end
+
+      # Reference implementation parses bundled directive supplements with
+      # month-bearing dates nondeterministically across load orders (a
+      # spurious day component appears). Quarantine deterministically.
+      def bundled_month_bug?(hash)
+        hash.to_s.include?("bundled_identifier") && hash.to_s.include?("month")
+      end
+
+      def quarantine!(record)
+        record.delete("identifier")
+        record.delete("representations")
+        record.delete("roundtrip")
+        record.delete("non_normalized_aliases")
+        record["notes"] = "nondeterministic parse in reference " \
+                          "implementation (month/day divergence) - " \
+                          "reference bug, quarantined from gates"
+      end
+
+      def round_trips?(hash)
+        @flavor_module::Identifier.from_hash(hash).to_hash == hash
+      rescue StandardError
+        false
+      end
+
+      def safe_urn(identifier)
+        identifier.to_urn
+      rescue StandardError
+        nil
+      end
+
+      def build_negative_cases(results)
+        cases = []
+        fail_files.each do |path|
+          type = File.basename(path, ".txt")
+          fail_fixture_lines(path).each_with_index do |input, index|
+            cases << negative_case(format("%s.neg.%s.%04d",
+                                          @flavor, type, index + 1), input)
+            results[:negative] += 1
+          end
+        end
+        cases
+      end
+
+      def negative_case(id, input)
+        @flavor_module.parse(input)
+        { "id" => id, "input" => input, "style" => Conformance.style_for(input),
+          "notes" => "fail fixture unexpectedly parsed - reclassify" }
+      rescue StandardError => e
+        { "id" => id, "input" => input,
+          "expect" => { "error" =>
+                        { "code" => Conformance.error_code_for(e) } } }
+      end
+
+      def write_yaml(path, entries)
+        File.write(path, YAML.dump(entries))
+      end
+    end
+  end
+end

data/lib/pubid/conformance/pending.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Implementation-side PENDING markers: corpus cases this implementation
+    # explicitly cannot handle yet. The testsuite owns expectations; this
+    # file owns the admission that they are not met - so a pending case is
+    # visible debt, never a silent failure and never a hidden pass.
+    #
+    # Format (conformance/pending.yaml at the implementation repo root):
+    #   "<case-id glob>":
+    #     reason: <one line, the root cause>
+    #     since: <YYYY-MM-DD>
+    #     ref: <TODO.correct-test-suite PR queue entry or contract id>
+    #
+    # Rules enforced elsewhere: a pending case that passes raises the
+    # "pending satisfied - remove the marker" alarm; entries without a ref
+    # are rejected at load.
+    class Pending
+      PATH = File.expand_path("../../../conformance/pending.yaml", __dir__)
+
+      class << self
+        def for(case_id)
+          entry = registry[case_id]
+          return entry if entry
+
+          registry.each do |pattern, meta|
+            next if pattern == case_id
+
+            return meta if File.fnmatch(pattern, case_id)
+          end
+          nil
+        end
+
+        def empty?
+          registry.empty?
+        end
+
+        private
+
+        def registry
+          @registry ||= load_registry
+        end
+
+        def load_registry
+          return {} unless File.exist?(PATH)
+
+          (YAML.safe_load_file(PATH) || {})
+            .to_h { |pattern, meta| [pattern, validated(meta, pattern)] }
+        end
+
+        def validated(meta, pattern)
+          unless meta.is_a?(Hash) && meta["reason"] && meta["ref"]
+            raise ArgumentError,
+                  "pending entry #{pattern.inspect} needs reason and ref"
+          end
+
+          meta
+        end
+      end
+    end
+  end
+end