pubid 2.0.0.pre.alpha.9 → 2.0.0.pre.alpha.12

data/lib/pubid/conformance/generator.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Migrates the ground-truth fixture library (spec/fixtures/{flavor}/
+    # identifiers/{pass,fail}/*.txt) into the neutral corpus by running the
+    # reference implementation. Fixture lines that normalize to the same
+    # identifier collapse into one canonical case carrying the old
+    # spellings as non_normalized_aliases (old => normalized, and the
+    # normalized form round-trips). Unparseable pass lines are recorded as
+    # visible debt; fail lines (stored as "#ID# Error: ..." comments in the
+    # fixture files) become negative cases. Nothing is sampled, filtered,
+    # or dropped.
+    class Generator
+      attr_reader :flavor
+
+      def initialize(flavor)
+        require "pubid"
+
+        @flavor = flavor.to_s
+        Pubid.eager_load_flavors!
+        @flavor_module = Pubid::Registry.get(@flavor)
+        raise ArgumentError, "unknown flavor #{@flavor}" unless @flavor_module
+      end
+
+      def generate(output_dir:)
+        results = { files: 0, cases: 0, negative: 0, debt: 0, aliases: 0,
+                    duplicates: 0, reclassify: 0, roundtrip_failures: 0 }
+        debt = []
+        pass_files.each do |path|
+          type = File.basename(path, ".txt")
+          cases, type_debt = build_pass_cases(path, type, results)
+          debt.concat(type_debt)
+          next if cases.empty?
+
+          write_yaml(File.join(output_dir, "#{type}.yaml"), cases)
+          results[:files] += 1
+        end
+        results[:debt] = debt.size
+        write_yaml(File.join(output_dir, "_debt.yaml"), debt) if debt.any?
+        negative = build_negative_cases(results)
+        if negative.any?
+          write_yaml(File.join(output_dir, "_negative.yaml"), negative)
+        end
+        results
+      end
+
+      private
+
+      def pass_files
+        Dir[File.join(fixtures_dir, "identifiers", "pass", "*.txt")] +
+          Dir[File.join(fixtures_dir, "identifiers", "full", "*.txt")]
+      end
+
+      def fail_files
+        Dir[File.join(fixtures_dir, "identifiers", "fail", "*.txt")]
+      end
+
+      def fixtures_dir
+        File.join(tests_repo, "reference-docs", @flavor)
+      end
+
+      def tests_repo
+        @tests_repo ||= ENV.fetch("PUBID_TESTSUITE_PATH",
+                                  File.expand_path("../../../../pubid-testsuite",
+                                                   __dir__))
+      end
+
+      def fixture_lines(path)
+        File.readlines(path).map(&:strip).reject do |line|
+          line.empty? || line.start_with?("#")
+        end
+      end
+
+      # Fail fixtures store identifiers as comment lines shaped
+      # "#IDENTIFIER# Parslet::ParseFailed: ...". Strip the leading # and
+      # the trailing historical annotation to recover the input.
+      def fail_fixture_lines(path)
+        File.readlines(path).filter_map do |raw|
+          line = raw.strip
+          next if line.empty?
+          next line unless line.start_with?("#")
+
+          input = line.sub(/\A#/, "").split("#", 2).first.to_s.strip
+          input unless input.empty?
+        end
+      end
+
+      def build_pass_cases(path, type, results)
+        groups = {}
+        debt = []
+        fixture_lines(path).each do |input|
+          record = build_record(input)
+          if record.key?(:debt)
+            debt << record[:debt]
+            next
+          end
+          key = JSON.generate(record[:case]["identifier"]) +
+            record[:case]["representations"]["human"]
+          (groups[key] ||= { record: record[:case], 
+                             inputs: [] })[:inputs] << input
+        end
+        cases = assign_ids(groups, type, results)
+        [cases, debt]
+      end
+
+      def assign_ids(groups, type, results)
+        groups.values.sort_by { |g| canonical_input(g) }.each_with_index
+          .map do |group, index|
+          record = group[:record]
+          record["id"] = format("%s.%s.%04d", @flavor, type, index + 1)
+          canonical = canonical_input(group)
+          record["style"] = Conformance.style_for(canonical)
+          inputs = group[:inputs]
+          duplicates = [inputs.count(canonical) - 1, 0].max
+          results[:duplicates] += duplicates
+          aliases = inputs.reject { |i| i == canonical }.uniq.sort
+          results[:aliases] += aliases.size
+          unless aliases.empty?
+            record["non_normalized_aliases"] = aliases.map do |a|
+              { "input" => a, "style" => Conformance.style_for(a) }
+            end
+          end
+          results[:roundtrip_failures] += 1 if record["roundtrip"] == false
+          results[:cases] += 1
+          record
+        end
+      end
+
+      def canonical_input(group)
+        group.dig(:record, "representations", "human")
+      end
+
+      def build_record(input)
+        identifier = @flavor_module.parse(input)
+        hash = identifier.to_hash
+        representations = { "human" => identifier.to_s }
+        urn = safe_urn(identifier)
+        representations["urn"] = urn if urn
+        record = { "id" => nil,
+                   "identifier" => Conformance.plainify(hash) }
+        record["representations"] = representations
+        record["roundtrip"] = false unless round_trips?(hash)
+        quarantine!(record) if bundled_month_bug?(hash)
+        { case: record }
+      rescue StandardError => e
+        { debt: { "id" => "#{@flavor}.debt.#{input.hash.abs}",
+                  "input" => input,
+                  "expect" => { "error" =>
+                                { "code" => Conformance.error_code_for(e) } },
+                  "notes" => "unparsed ground-truth fixture: " \
+                             "#{e.class}: #{e.message.to_s[0, 200]}" } }
+      end
+
+      # Reference implementation parses bundled directive supplements with
+      # month-bearing dates nondeterministically across load orders (a
+      # spurious day component appears). Quarantine deterministically.
+      def bundled_month_bug?(hash)
+        hash.to_s.include?("bundled_identifier") && hash.to_s.include?("month")
+      end
+
+      def quarantine!(record)
+        record.delete("identifier")
+        record.delete("representations")
+        record.delete("roundtrip")
+        record.delete("non_normalized_aliases")
+        record["notes"] = "nondeterministic parse in reference " \
+                          "implementation (month/day divergence) - " \
+                          "reference bug, quarantined from gates"
+      end
+
+      def round_trips?(hash)
+        @flavor_module::Identifier.from_hash(hash).to_hash == hash
+      rescue StandardError
+        false
+      end
+
+      def safe_urn(identifier)
+        identifier.to_urn
+      rescue StandardError
+        nil
+      end
+
+      def build_negative_cases(results)
+        cases = []
+        fail_files.each do |path|
+          type = File.basename(path, ".txt")
+          fail_fixture_lines(path).each_with_index do |input, index|
+            cases << negative_case(format("%s.neg.%s.%04d",
+                                          @flavor, type, index + 1), input)
+            results[:negative] += 1
+          end
+        end
+        cases
+      end
+
+      def negative_case(id, input)
+        @flavor_module.parse(input)
+        { "id" => id, "input" => input, "style" => Conformance.style_for(input),
+          "notes" => "fail fixture unexpectedly parsed - reclassify" }
+      rescue StandardError => e
+        { "id" => id, "input" => input,
+          "expect" => { "error" =>
+                        { "code" => Conformance.error_code_for(e) } } }
+      end
+
+      def write_yaml(path, entries)
+        File.write(path, YAML.dump(entries))
+      end
+    end
+  end
+end

data/lib/pubid/conformance/pending.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Implementation-side PENDING markers: corpus cases this implementation
+    # explicitly cannot handle yet. The testsuite owns expectations; this
+    # file owns the admission that they are not met - so a pending case is
+    # visible debt, never a silent failure and never a hidden pass.
+    #
+    # Format (conformance/pending.yaml at the implementation repo root):
+    #   "<case-id glob>":
+    #     reason: <one line, the root cause>
+    #     since: <YYYY-MM-DD>
+    #     ref: <TODO.correct-test-suite PR queue entry or contract id>
+    #
+    # Rules enforced elsewhere: a pending case that passes raises the
+    # "pending satisfied - remove the marker" alarm; entries without a ref
+    # are rejected at load.
+    class Pending
+      PATH = File.expand_path("../../../conformance/pending.yaml", __dir__)
+
+      class << self
+        def for(case_id)
+          entry = registry[case_id]
+          return entry if entry
+
+          registry.each do |pattern, meta|
+            next if pattern == case_id
+
+            return meta if File.fnmatch(pattern, case_id)
+          end
+          nil
+        end
+
+        def empty?
+          registry.empty?
+        end
+
+        private
+
+        def registry
+          @registry ||= load_registry
+        end
+
+        def load_registry
+          return {} unless File.exist?(PATH)
+
+          (YAML.safe_load_file(PATH) || {})
+            .to_h { |pattern, meta| [pattern, validated(meta, pattern)] }
+        end
+
+        def validated(meta, pattern)
+          unless meta.is_a?(Hash) && meta["reason"] && meta["ref"]
+            raise ArgumentError,
+                  "pending entry #{pattern.inspect} needs reason and ref"
+          end
+
+          meta
+        end
+      end
+    end
+  end
+end

data/lib/pubid/conformance/runner.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Pubid
+  module Conformance
+    # Executes the pubid-testsuite corpus against this implementation.
+    # CLEAN flavors hard-gate (failures abort); DIRTY flavors report as
+    # the known reference-defect ledger.
+    class Runner
+      REGISTRY_KEYS = { "tgpp" => "3gpp" }.freeze
+
+      def run(flavors = corpus_flavors)
+        require "pubid"
+
+        if flavors.empty?
+          raise ArgumentError, "no corpus flavors found at #{corpus_dir} - " \
+                               "refusing to report vacuous success"
+        end
+
+        Pubid.eager_load_flavors!
+        flavors.each_with_object([]) do |flavor, failures|
+          failures.concat(run_flavor(flavor))
+        end
+      end
+
+      private
+
+      def corpus_flavors
+        Dir[File.join(corpus_dir, "*")].select { |p| File.directory?(p) }
+          .map { |p| File.basename(p) }
+      end
+
+      def corpus_dir
+        File.join(ENV.fetch("PUBID_TESTSUITE_PATH",
+                            File.expand_path("../../../../pubid-testsuite",
+                                             __dir__)), "tests")
+      end
+
+      def run_flavor(flavor)
+        flavor_module = Pubid::Registry.get(
+          REGISTRY_KEYS.fetch(flavor, flavor),
+        )
+        raise ArgumentError, "unknown flavor #{flavor}" if flavor_module.nil?
+
+        stats = Hash.new(0)
+        failures = []
+        each_payload_case(flavor) do |test_case|
+          execute(test_case, flavor_module, stats, failures)
+        end
+        Corpus.negative_file(flavor, corpus_dir).each do |path|
+          Corpus.load_file(path).each do |test_case|
+            execute_negative(test_case, flavor_module, stats, failures)
+          end
+        end
+        report(flavor, stats, failures)
+        known_dirty?(flavor) ? [] : failures
+      end
+
+      def each_payload_case(flavor, &block)
+        Corpus.case_files(flavor, corpus_dir).each do |path|
+          Corpus.load_file(path).each(&block)
+        end
+      end
+
+      def known_dirty?(flavor)
+        status = File.join(corpus_dir, flavor, "_status.yaml")
+        File.exist?(status) && !YAML.safe_load_file(status)["clean"]
+      end
+
+      # Every corpus case lands in exactly one quadrant:
+      #   pass | FAIL (not pending, breaks the CLEAN gate) | pending
+      #   (explicitly not handled yet, see Conformance::Pending) | review
+      #   (the expectation itself is flagged for human verification).
+      def execute(test_case, flavor_module, stats, failures)
+        id = test_case.id
+        return stats[:review] += 1 if test_case.review?
+
+        if Pending.for(id)
+          run_pending(id, test_case, flavor_module, stats, failures)
+        else
+          run_case(test_case, flavor_module, stats, failures)
+        end
+      end
+
+      # Pending cases still run: a pending case that PASSES is reported as
+      # pending-satisfied - a cleanup alarm, the marker must be removed.
+      def run_pending(id, test_case, flavor_module, stats, failures)
+        stats[:pending] += 1
+        before = failures.size
+        run_case(test_case, flavor_module, stats, failures)
+        return failures.slice!(before..) if failures.size > before
+
+        stats[:pending_satisfied] += 1
+        puts "  PENDING-SATISFIED #{id} - remove the marker"
+      end
+
+      def run_case(test_case, flavor_module, stats, failures)
+        if test_case.error_case?
+          execute_error_case(test_case, flavor_module, stats, failures)
+          return
+        end
+        return stats[:quarantined] += 1 if test_case.quarantined?
+
+        stats[:cases] += 1
+        Checks.check_case(test_case, flavor_module).each do |mismatch|
+          stat_for(mismatch, stats)
+          failures << mismatch
+        end
+      end
+
+      def stat_for(mismatch, stats)
+        case mismatch
+        when / raised\z/ then stats[:fail_parse] += 1
+        when /canonical hash/ then stats[:fail_tree] += 1
+        when / human\z/, / urn\z/
+          stats[:"fail_#{mismatch.split.last}"] += 1
+        when / alias/ then stats[:fail_alias] += 1
+        else stats[:fail_roundtrip] += 1
+        end
+      end
+
+      # Negatives (fail fixtures decoded by the exporter). A row with
+      # expect.error must be rejected by the reference; a row without it
+      # is the exporter's reclassification alarm - the fail-fixture line
+      # now parses, so the fixture belongs in pass/ - reported, never
+      # gated, like the PENDING-SATISFIED alarm.
+      def execute_negative(test_case, flavor_module, stats, failures)
+        return stats[:reclass] += 1 unless test_case.error_case?
+
+        execute_error_case(test_case, flavor_module, stats, failures)
+      end
+
+      def execute_error_case(test_case, flavor_module, stats, failures)
+        stats[:error_cases] += 1
+        flavor_module.parse(test_case.input)
+        stats[:fail_error] += 1
+        failures << "#{test_case.id} unexpectedly parsed"
+      rescue StandardError
+        stats[:error_ok] += 1
+      end
+
+      def report(flavor, stats, failures)
+        puts format(
+          "%-6s cases=%-6d err=%-4d quar=%-3d fail=%-4d " \
+          "pend=%-4d pendok=%-3d reclass=%-4d review=%d",
+          flavor, stats[:cases], stats[:error_cases], stats[:quarantined],
+          failures.size, stats[:pending], stats[:pending_satisfied],
+          stats[:reclass], stats[:review]
+        )
+        failures.first(10).each { |f| puts "  FAIL #{f}" }
+      end
+    end
+  end
+end

data/lib/pubid/conformance.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "json"
+
+module Pubid
+  # The shared, language-independent conformance corpus (conformance/).
+  # The identifier structure in each case is THE GEM'S OWN canonical
+  # to_hash, embedded verbatim (the same wire format relaton persists).
+  # Nothing is reinvented: the corpus adds only what the gem cannot know -
+  # representations, normalization aliases, styling, and neutral error
+  # codes for debt/negative records.
+  module Conformance
+    autoload :Checks, "pubid/conformance/checks"
+    autoload :Corpus, "pubid/conformance/corpus"
+    autoload :Generator, "pubid/conformance/generator"
+    autoload :Pending, "pubid/conformance/pending"
+    autoload :Runner, "pubid/conformance/runner"
+
+    # Neutral error vocabulary. Each implementation maps its native
+    # exceptions onto these codes; the corpus never records Ruby classes.
+    ERROR_CODES = {
+      "Parslet::ParseFailed" => "parse_failed",
+      "ArgumentError" => "invalid_argument",
+    }.freeze
+
+    class << self
+      def error_code_for(exception)
+        ERROR_CODES.fetch(exception.class.name, "unclassified")
+      end
+
+      # Heuristic PubID styling version. v1 = legacy "ISO/R 2533/3-1975";
+      # v2 = dotted/uppercase ("AMD.1"); v3 = modern ("Amd 1"). Heuristic
+      # until the model exposes its parsed format (TODO.restructure/12).
+      def style_for(input)
+        return "v1" if /\bISO\/R\b|\bISO R\b/.match?(input)
+        return "v2" if /\b(?:Amd|AMD|Cor|COR|Suppl|SUPPL)\.\d/.match?(input)
+
+        "v3"
+      end
+
+      # Hygiene only (not a redesign): guarantee the corpus is pure YAML -
+      # component objects leaking through serialization are reduced to
+      # their string rendering.
+      def plainify(value)
+        case value
+        when Hash then value.to_h { |key, entry| [key.to_s, plainify(entry)] }
+        when Array then value.map { |entry| plainify(entry) }
+        when String, Integer, Float, TrueClass, FalseClass, NilClass then value
+        else value.to_s
+        end
+      end
+    end
+  end
+end

data/lib/pubid/core.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
+require "pubid"
 module Pubid
   # Core module for shared functionality across all flavors.
   module Core
     autoload :UpdateCodes, "#{__dir__}/core/update_codes"
+    autoload :PatternDocGenerator, "#{__dir__}/core/pattern_doc_generator"
   end
 end

data/lib/pubid/csa/builder.rb

@@ -101,7 +101,7 @@ module Pubid
           else
             begin
               Pubid::Iso.parse(base_str)
-            rescue Pubid::Errors::ParseError
+            rescue Parslet::ParseFailed
               Pubid::Iec.parse(base_str)
             end
           end
@@ -128,17 +128,17 @@ module Pubid
           end
           amendment_str = "#{base_str}/Amd #{parsed_hash[:adoption_amendment]}:#{wrapped_amendment_year}"
 
-          wrapped_identifier = if adoption_org.start_with?("ISO", "CEI")
-                                 Pubid::Iso.parse(amendment_str)
-                               elsif adoption_org.start_with?("IEC")
-                                 Pubid::Iec.parse(amendment_str)
-                               else
-                                 begin
-                                   Pubid::Iso.parse(amendment_str)
-                                 rescue Pubid::Errors::ParseError
-                                   Pubid::Iec.parse(amendment_str)
-                                 end
-                               end
+          base = if adoption_org.start_with?("ISO", "CEI")
+                   Pubid::Iso.parse(amendment_str)
+                 elsif adoption_org.start_with?("IEC")
+                   Pubid::Iec.parse(amendment_str)
+                 else
+                   begin
+                     Pubid::Iso.parse(amendment_str)
+                   rescue Parslet::ParseFailed
+                     Pubid::Iec.parse(amendment_str)
+                   end
+                 end
         else
           # No amendment, simple base identifier
           wrapped_id_str = "#{adoption_org} #{adoption_number}#{year_sep}#{wrapped_year}"
@@ -148,20 +148,20 @@ module Pubid
             wrapped_id_str = wrapped_id_str.sub("CEI/IEC", "IEC")
           end
 
-          wrapped_identifier = if adoption_org.start_with?("ISO", "CEI")
-                                 Pubid::Iso.parse(wrapped_id_str)
-                               elsif adoption_org.start_with?("IEC")
-                                 Pubid::Iec.parse(wrapped_id_str)
-                               else
-                                 begin
-                                   Pubid::Iso.parse(wrapped_id_str)
-                                 rescue Pubid::Errors::ParseError
-                                   Pubid::Iec.parse(wrapped_id_str)
-                                 end
-                               end
+          base = if adoption_org.start_with?("ISO", "CEI")
+                   Pubid::Iso.parse(wrapped_id_str)
+                 elsif adoption_org.start_with?("IEC")
+                   Pubid::Iec.parse(wrapped_id_str)
+                 else
+                   begin
+                     Pubid::Iso.parse(wrapped_id_str)
+                   rescue Parslet::ParseFailed
+                     Pubid::Iec.parse(wrapped_id_str)
+                   end
+                 end
         end
 
-        adoption.wrapped_identifier = wrapped_identifier
+        adoption.base = base
 
         # Handle reaffirmation
         if parsed_hash[:reaffirmation]
@@ -195,23 +195,25 @@ module Pubid
           # SERIES matched - build Series identifier with CAN/CSA- prefix
           # The Series.to_s will handle rendering as "CAN/CSA-A220 SERIES-06"
           # (keeping the full prefix instead of "CAN/CSA-A220...")
-          wrapped_identifier = build_series(wrapped_data)
+          base = build_series(wrapped_data)
           # Set the full publisher_prefix on the wrapped Series for proper rendering
-          # The test expects wrapped_identifier.publisher_prefix to be "CAN/CSA-"
+          # The test expects base.publisher_prefix to be "CAN/CSA-"
         elsif parsed_hash.key?(:cec_part)
           # CEC matched - build Cec identifier
-          wrapped_identifier = build_cec(wrapped_data)
+          base = build_cec(wrapped_data)
         else
           # Standard CSA identifier
-          wrapped_identifier = build_single(wrapped_data)
+          base = build_single(wrapped_data)
         end
-        wrapped_identifier.publisher_prefix = original_prefix if original_prefix && wrapped_identifier.class.attributes.key?(:publisher_prefix)
+        base.publisher_prefix = original_prefix if original_prefix && base.class.attributes.key?(:publisher_prefix)
 
-        canadian_adopted.wrapped_identifier = wrapped_identifier
+        canadian_adopted.base = base
 
-        # Handle reaffirmation - supports both 2-digit and 4-digit years
-        # Note: CanadianAdopted (WrapperIdentifier) doesn't track original_reaffirmation_4digit
-        # That's only for Series/Cec identifiers
+        # Handle reaffirmation - supports both 2-digit and 4-digit years.
+        # The wrapper's own original_reaffirmation_4digit is left at its
+        # default: CanadianAdopted#to_s only consults it when the wrapped id
+        # carries no reaffirmation of its own, and Series/Cec track the printed
+        # width themselves.
         if parsed_hash[:reaffirmation_4digit]
           # Original was 4-digit (e.g., "R2019")
           canadian_adopted.reaffirmation = parsed_hash[:reaffirmation_4digit].to_s
@@ -251,7 +253,7 @@ module Pubid
 
         # Code
         if parsed_hash[:code]
-          series.code = Components::Code.new(value: parsed_hash[:code].to_s)
+          series.number = parsed_hash[:code].to_s
         end
 
         # Year format and year
@@ -451,10 +453,10 @@ module Pubid
       def build_combined(parsed_hash)
         combined = Identifiers::Combined.new
 
-        # Build each part
-        combined.first = build_single(parsed_hash[:first])
-        combined.second = build_single(parsed_hash[:second])
-        combined.third = build_single(parsed_hash[:third]) if parsed_hash[:third]
+        # Build each designation, in printed order, into one collection.
+        combined.identifiers = %i[first second third]
+          .filter_map { |key| parsed_hash[key] }
+          .map { |part| build_single(part) }
 
         # Detect separator from parser marker
         combined.separator = if parsed_hash[:comma_separator]
@@ -495,15 +497,13 @@ module Pubid
         identifier_class = select_identifier_class(data)
         identifier = identifier_class.new
 
-        # Publisher prefix (CAN/CSA-, CAN3-, or CSA)
-        # For code_only identifiers (no prefix in original), set to empty string
-        # to prevent default "CSA" from being added in rendering
+        # Publisher prefix (CAN/CSA-, CAN3-, or CSA).
+        # A code with no prefix in the original is a code_only identifier; the
+        # flag stops rendering from supplying the default "CSA".
         if data[:publisher_prefix]
           identifier.publisher_prefix = data[:publisher_prefix].to_s
-        elsif data[:code] && !data[:publisher_prefix]
-          # Code exists but no publisher_prefix = code_only identifier
-          # Use empty string to indicate "no prefix wanted"
-          identifier.publisher_prefix = ""
+        elsif data[:code]
+          identifier.code_only = true
         end
 
         # Publisher presence flag
@@ -520,13 +520,13 @@ module Pubid
           # Pattern: "C22.1-15" should become code="C22.1", year="2015"
           if !data[:year] && code_value =~ /^(.+)-(\d{2})$/
             # Split code and year
-            identifier.code = Components::Code.new(value: $1)
+            identifier.number = $1
             # Convert 2-digit year to 4-digit
             year_2digit = $2
             identifier.year = "20#{year_2digit}"
             identifier.year_format = "dash"
           else
-            identifier.code = Components::Code.new(value: code_value)
+            identifier.number = code_value
           end
         end