relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/iala/bibliography.rb

@@ -0,0 +1,146 @@
+require "net/http"
+
+module Relaton
+  module Iala
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-iala/main/".freeze
+
+      class << self
+        # Search for an IALA publication by its identifier.
+        #
+        # @param text [String] the IALA reference to look up (e.g. "IALA S1070")
+        # @param _year [String, nil] optional edition/year filter
+        # @param _opts [Hash] options (unused)
+        # @return [Relaton::Iala::Item, nil]
+        def search(text, _year = nil, _opts = {})
+          Util.info "Fetching from Relaton repository ...", key: text
+          row = best_match text
+          unless row
+            Util.info "Not found.", key: text
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          item = Relaton::Iala::Item.from_yaml resp.body
+          Util.info "Found: `#{item.docidentifier.first&.content}`", key: text
+          item.tap { |i| i.fetched = Date.today.to_s }
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+               Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+               Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
+               Errno::ETIMEDOUT => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        #
+        # Find the index row for a reference, newest edition first.
+        #
+        # Passing the pubid to `Index::Type#search` is what enables the binary
+        # search on `id.root.number` — with a block alone, or with the plain
+        # string this used to pass, the whole index is scanned. Selection is
+        # `Type#search`'s default with no block — pubid's asymmetric subset
+        # match (`Pubid::SubsetMatch`): the reference on the left, the row on
+        # the right, and a component the reference omits matches any value.
+        # `edition` and `language` are IALA's only
+        # optional components, so a bare `IALA M0001` still finds the
+        # `Ed 9.0 (E)` row. The document type is deliberately NOT a wildcard —
+        # it is the identifier's class, and `===` requires the same class. 17 of
+        # the 309 index numbers are shared by two types, so `R1001` and `C1001`
+        # must never match each other. An `Annex` row carries its base
+        # identifier, which `===` compares with its own subset match.
+        #
+        # @param text [String]
+        # @return [Hash, nil]
+        #
+        # The substring-scan fallback this used to take when parsing failed is
+        # gone with the rescue. It was the same conflation the convention
+        # rejects: an unparseable string matched a SUBSTRING of the rendered
+        # ids, so an ambiguous reference silently resolved to whichever row
+        # sorted first, instead of telling the caller the reference is not an
+        # identifier. ecma, w3c and xsf never had one.
+        def best_match(text)
+          pubid = parse_ref text
+          rows = index.search(pubid)
+          rows.max_by { |r| [edition_key(r[:id].edition), language_key(r[:id]), r[:file]] }
+        end
+
+        #
+        # Order key for an IALA edition.
+        #
+        # Editions are dotted version strings, so they must be compared segment
+        # by segment as integers rather than as text: `"10.0"` is newer than
+        # `"9.0"` but sorts before it as a string. No published edition has a
+        # segment above 9 today, so this is a latent bug rather than a live one
+        # — the same one already fixed for ETSI. An absent edition sorts below
+        # every present one.
+        #
+        # `2` and `2.0` are distinct editions in the index and compare as
+        # `[2] < [2, 0]`; `2.02` and `2.2` reduce to the same segments. Both
+        # pairs are broken by the edition text and then the file path, so a
+        # repeated lookup returns the same document (the index sort is not
+        # stable).
+        #
+        # @param edition [String, nil]
+        # @return [Array]
+        #
+        def edition_key(edition)
+          [edition.to_s.split(".").map(&:to_i), edition.to_s]
+        end
+
+        # Prefer the language-neutral record over its translations: it is the
+        # base document, and the translations are keyed off it.
+        #
+        # @param row_id [Pubid::Iala::Identifier]
+        # @return [Integer]
+        #
+        def language_key(row_id)
+          row_id.language.nil? ? 1 : 0
+        end
+
+        #
+        # Parse a user reference into a `Pubid::Iala::Identifier`, or nil.
+        #
+        # `Pubid::Iala` takes the `IALA ` publisher prefix as optional and
+        # zero-pads the number to its type's canonical width, so `IALA M1`,
+        # `M0001` and `R1016:ed2.0(F)` all parse without normalization here.
+        #
+        # @param text [String]
+        # @return [Pubid::Iala::Identifier, nil]
+        #
+        # An unrecognized reference **raises**; like ISO, ETSI and 3GPP we let it
+        # propagate. relaton-cli rescues `Pubid::Errors::Error` and renders
+        # `"..." is not a recognized standards identifier`
+        # (`gems/relaton-cli/lib/relaton/cli/command.rb:324`), and `Db#fetch`
+        # logs it through the `StandardError` arm at `lib/relaton/db.rb:122`.
+        # Rescuing here would collapse "this identifier is malformed" into "no
+        # such document", leaving a caller unable to tell them apart.
+        def parse_ref(text)
+          ::Pubid::Iala::Identifier.parse text.to_s.strip
+        end
+
+        # The index is pubid-backed: `pubid_class:` is what makes
+        # Relaton::Index deserialize the rows into identifiers, sort them by
+        # `id.root.number`, and let `Type#search` bsearch. Without it the rows
+        # stay raw hashes and every lookup scans all 701 of them.
+        def index
+          Relaton::Index.find_or_create(
+            :iala,
+            url: "#{ENDPOINT}#{INDEXFILE}.zip",
+            file: "#{INDEXFILE}.yaml",
+            pubid_class: ::Pubid::Iala::Identifier,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/relaton/iala/docidentifier.rb

@@ -0,0 +1,89 @@
+module Relaton
+  module Iala
+    # Structured IALA document identifier. The docid string is the canonical
+    # form (e.g. "IALA S1070 Ed 2.0"). Defined as a subclass so future
+    # Pubid::Iala integration can hook in via #pubid without changing the
+    # public interface.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      # Capture the inherited (LocalizedMarkedUpString) content setter before
+      # overriding #content=, so #refresh_content! can write the re-rendered
+      # string back WITHOUT re-parsing (a re-parse would rebuild @pubid from
+      # the string, which may not even parse back — e.g. an all-parts
+      # wrapper's own "(all parts)" rendering — and discard the mutation).
+      alias_method :store_content, :content=
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        return unless value
+
+        @pubid = begin
+          # `pubid/iala` needs pubid core (Pubid::PrefixesSupport) loaded first,
+          # so require both. Rescue LoadError so a missing pubid gem degrades to
+          # a plain string, and StandardError so non-IALA/unparseable content
+          # just leaves @pubid nil rather than raising.
+          require "pubid"
+          require "pubid/iala"
+          ::Pubid::Iala.parse(value)
+        rescue LoadError, StandardError
+          nil
+        end
+      end
+
+      # IALA's Pubid::Iala::Identifier models only publisher/number/edition/
+      # language. It has no distinct `date` component and folds any subpart
+      # inline into `number` (e.g. "0103-1"), so:
+      #   * #remove_date! maps onto `edition` — IALA's version discriminator —
+      #     yielding the version-agnostic ("most recent") reference.
+      #   * #remove_part! clears the (currently-unused) part/subpart attributes;
+      #     it is a no-op for the rendered content today, but is implemented so
+      #     it never raises and starts working automatically should pubid-iala
+      #     ever model `part` separately.
+      # All three no-op safely when @pubid is nil (parse failed or the pubid gem
+      # is unavailable), so Bib::ItemData#to_all_parts / #to_most_recent_reference
+      # never raise on IALA items.
+
+      def remove_part!
+        clear_attr!(:part)
+      end
+
+      def remove_date!
+        clear_attr!(:edition)
+      end
+
+      def to_all_parts!
+        return if !@pubid || @pubid.all_parts?
+
+        # `#exclude` (no args) rebuilds a full independent copy — `remove_part!`
+        # / `remove_date!` mutate `@pubid` in place, so a bare reference here
+        # would lose the original edition to that mutation too.
+        original = @pubid.exclude
+        remove_part!
+        remove_date!
+        @pubid = original.to_all_parts
+      end
+
+      private
+
+      def clear_attr!(attr)
+        return unless @pubid && @pubid.respond_to?("#{attr}=")
+
+        @pubid.public_send("#{attr}=", nil)
+        @pubid.subpart = nil if attr == :part && @pubid.respond_to?(:subpart=)
+        refresh_content!
+      end
+
+      def refresh_content!
+        store_content(@pubid.to_s) if @pubid
+      end
+    end
+  end
+end

data/lib/relaton/iala/doctype.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Iala
+    # IALA publication document types. The value is carried in the
+    # inherited `content` attribute (e.g. `standard`, `recommendation`).
+    # The list mirrors the seven categories the IALA site exposes.
+    class Doctype < Bib::Doctype
+      TYPES = %w[
+        standard
+        recommendation
+        guideline
+        manual
+        model-course
+        report
+        resolution
+      ].freeze
+    end
+  end
+end

data/lib/relaton/iala/ext.rb

@@ -0,0 +1,32 @@
+module Relaton
+  module Iala
+    # IALA bibliographic item extension. Adds IALA-specific structured
+    # metadata on top of the shared Relaton::Bib::Ext so the fields
+    # round-trip through both XML and YAML without per-repo merge hacks.
+    # Doctype is referenced lazily via autoload from lib/relaton/iala.rb.
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+      attribute :urn,        :string
+      attribute :webpage,    :string
+      attribute :committee,  :string
+      attribute :normative,  :string
+      attribute :supersedes, :string, collection: true, initialize_empty: true
+
+      xml do
+        map_element "urn",        to: :urn
+        map_element "webpage",    to: :webpage
+        map_element "committee",  to: :committee
+        map_element "normative",  to: :normative
+        map_element "supersedes", to: :supersedes
+      end
+
+      key_value do
+        map_element "urn",        to: :urn
+        map_element "webpage",    to: :webpage
+        map_element "committee",  to: :committee
+        map_element "normative",  to: :normative
+        map_element "supersedes", to: :supersedes
+      end
+    end
+  end
+end

data/lib/relaton/iala/item.rb

@@ -0,0 +1,21 @@
+module Relaton
+  module Iala
+    # IALA bibliographic item. Re-declares +ext+ to use the typed Ext
+    # subclass so IALA-specific fields round-trip natively. Sub-files
+    # (ItemData, Ext, Docidentifier, Relation) are loaded lazily via the
+    # autoload entries declared in lib/relaton/iala.rb.
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end
+
+# Wire up the back-reference: an Item's relation collection contains
+# Relation nodes whose bibitem is an ItemBase. Referencing
+# Relaton::Iala::Relation triggers its autoload.
+Relaton::Iala::Item.attribute :relation, Relaton::Iala::Relation,
+                              collection: true, initialize_empty: true

data/lib/relaton/iala/item_base.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Iala
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end

data/lib/relaton/iala/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Iala
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/iala/processor.rb

@@ -0,0 +1,43 @@
+require "relaton/core/processor"
+
+module Relaton
+  module Iala
+    class Processor < Core::Processor
+      attr_reader :idtype
+
+      def initialize
+        @short = :relaton_iala
+        @prefix = "IALA"
+        @defaultprefix = %r{^IALA\s}
+        @idtype = "IALA"
+      end
+
+      def get(code, date, opts)
+        require_relative "../iala"
+        Bibliography.get(code, date, opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../iala"
+        Item.from_xml xml
+      end
+
+      def from_yaml(yaml)
+        require_relative "../iala"
+        Item.from_yaml(yaml)
+      end
+
+      def grammar_hash
+        require_relative "../iala"
+        @grammar_hash ||= ::Relaton::Iala.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../iala"
+        Relaton::Index.find_or_create(
+          :iala, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/iala/relation.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Iala
+    class Relation < Bib::Relation
+      attribute :bibitem, ItemBase
+    end
+  end
+end

data/lib/relaton/iala/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iala
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-iala".freeze
+    end
+  end
+end

data/lib/relaton/iala.rb

@@ -0,0 +1,35 @@
+require "pubid"
+require "relaton/bib"
+require "relaton/index"
+require "relaton/version"
+
+# Relaton::Iala is the IALA (International Organization for Marine Aids
+# to Navigation) flavor of the unified relaton gem. Adds typed Ext fields
+# (urn, webpage, committee, normative, supersedes) so they round-trip
+# natively through YAML and XML without per-repo merge hacks.
+module Relaton
+  module Iala
+    INDEXFILE = "index-v2".freeze
+
+    class Error < StandardError; end
+
+    autoload :Util,          "relaton/iala/util"
+    autoload :Doctype,       "relaton/iala/doctype"
+    autoload :Ext,           "relaton/iala/ext"
+    autoload :ItemData,      "relaton/iala/item_data"
+    autoload :ItemBase,      "relaton/iala/item_base"
+    autoload :Item,          "relaton/iala/item"
+    autoload :Relation,      "relaton/iala/relation"
+    autoload :Bibitem,       "relaton/iala/bibitem"
+    autoload :Bibdata,       "relaton/iala/bibdata"
+    autoload :Docidentifier, "relaton/iala/docidentifier"
+    autoload :Bibliography,  "relaton/iala/bibliography"
+    autoload :Processor,     "relaton/iala/processor"
+
+    # Returns hash of XML grammar
+    # @return [String]
+    def self.grammar_hash
+      Digest::MD5.hexdigest Relaton::VERSION
+    end
+  end
+end

data/lib/relaton/iana/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iana
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/iana/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iana
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/iana/bibliography.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Iana
+    # Retrieval front-end for the IANA flavor. Parses the citation with
+    # Pubid::Iana, looks it up in the pubid-structured `relaton-data-iana`
+    # index-v2, and fetches the matching per-document YAML over HTTP.
+    #
+    # An IANA entry is a protocol registry, not a numbered standard: the
+    # identifier is a registry slug with at most one "/"
+    # (`IANA _6lowpan-parameters`, `IANA _6lowpan-parameters/lowpan_nhc`). The
+    # index rows key on the TOP-LEVEL slug (`Identifiers::Registry#number`), so
+    # a registry and all of its sub-registries share one bsearch bucket and the
+    # block below distinguishes them.
+    module Bibliography
+      SOURCE = "https://raw.githubusercontent.com/relaton/relaton-data-iana/refs/heads/v2/"
+
+      # @param text [String, Pubid::Iana::Identifier]
+      # @return [Relaton::Iana::Item, nil]
+      def search(text)
+        pubid = parse_ref text
+        return unless pubid
+
+        # Pass the pubid so Relaton::Index narrows candidates by number via
+        # binary search before applying the block. The match is exact, so there
+        # is at most one row.
+        row = index.search(pubid) { |r| pubid_match? r[:id], pubid }.first
+        return unless row
+
+        fetch_row row
+      rescue SocketError, Timeout::Error, Errno::EINVAL, Errno::ECONNRESET,
+             EOFError, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, Errno::ETIMEDOUT => e
+        raise Relaton::RequestError, e.message
+      end
+
+      # @param ref [String] the IANA registry slug to look up
+      # @param year [String, NilClass] not used
+      # @param opts [Hash] options
+      # @return [Relaton::Iana::Item, nil]
+      def get(ref, _year = nil, _opts = {})
+        Util.info "Fetching from Relaton repository ...", key: ref
+        result = search(ref)
+        unless result
+          Util.info "Not found.", key: ref
+          return
+        end
+
+        Util.info "Found: `#{result.docidentifier[0].content}`", key: ref
+        result
+      end
+
+      private
+
+      def index
+        Relaton::Index.find_or_create :iana, url: "#{SOURCE}#{INDEXFILE}.zip",
+                                             file: "#{INDEXFILE}.yaml",
+                                             pubid_class: ::Pubid::Iana::Identifier
+      end
+
+      # `Pubid::Iana::Identifier.parse` accepts both the printed `IANA <slug>`
+      # form and the bare slug the index rows render back to. It raises a plain
+      # RuntimeError on anything else, and a citation relaton cannot parse is a
+      # miss, not an error — `#get` reports it as "Not found.".
+      #
+      # @param text [String, Pubid::Iana::Identifier]
+      # @return [Pubid::Iana::Identifier, nil]
+      def parse_ref(text)
+        return text unless text.is_a? String
+
+        ::Pubid::Iana::Identifier.parse text
+      rescue StandardError => e
+        Util.warn "Failed to parse pubid `#{text}`: #{e.message}"
+        nil
+      end
+
+      # Both arguments are Pubid::Iana identifiers. Compared field by field
+      # rather than with `==` so the two identity fields are explicit: a bare
+      # registry must NOT match its own sub-registries, which share its bsearch
+      # bucket. (index-v1 matched by substring, so `IANA rpki` also hit
+      # `rpki/signed-objects` and `min_by` picked the shortest id.)
+      def pubid_match?(row_id, query)
+        row_id.class == query.class &&
+          row_id.number == query.number &&
+          row_id.sub_registry.to_s == query.sub_registry.to_s
+      end
+
+      # @param row [Hash] an index row
+      # @return [Relaton::Iana::Item, nil]
+      def fetch_row(row)
+        resp = Net::HTTP.get_response URI("#{SOURCE}#{row[:file]}")
+        return unless resp.code == "200"
+
+        Item.from_yaml(resp.body).tap { |item| item.fetched = Date.today.to_s }
+      end
+
+      extend self
+    end
+  end
+end

data/lib/relaton/iana/data_fetcher.rb

@@ -0,0 +1,101 @@
+require "relaton/core"
+require_relative "../iana"
+require_relative "parser"
+
+module Relaton
+  module Iana
+    class DataFetcher < Core::DataFetcher
+      #
+      # Parse documents
+      #
+      def log_error(msg)
+        Util.error msg
+      end
+
+      def fetch(_source = nil)
+        Dir["iana-registries/**/*.xml"].each do |file|
+          content = File.read file, encoding: "UTF-8"
+          parse(content) if content.include? "<registry"
+        rescue StandardError => e
+          Util.error "Error: #{e.message}. File: #{file}"
+        end
+        index.save
+        report_errors
+      end
+
+      private
+
+      def index
+        @index ||= Relaton::Index.find_or_create :iana, file: "#{INDEXFILE}.yaml",
+                                                        pubid_class: ::Pubid::Iana::Identifier
+      end
+
+      #
+      # Add a document to the pubid-keyed index-v2.
+      #
+      # Stores the Pubid::Iana identifier object (not its hash): with
+      # pubid_class set, Relaton::Index sorts the index by the id's number on
+      # save and serializes each id to its `_type: pubid:iana:registry` hash.
+      #
+      # The rescue is load-bearing. Relaton::Index rejects the WHOLE index if a
+      # single row fails to deserialize, so a slug pubid cannot parse has to be
+      # skipped here rather than raise and abort the crawl.
+      #
+      # @param [String] docnumber bare registry slug, `registry[/sub-registry]`
+      # @param [String] file path of the document file
+      #
+      def add_to_index(docnumber, file)
+        index.add_or_update ::Pubid::Iana::Identifier.parse(docnumber), file
+      rescue StandardError => e
+        Util.warn "Skipping index entry for `#{docnumber}` (#{file}): #{e.message}"
+      end
+
+      def parse(content)
+        xml = Nokogiri::XML(content)
+        registry = xml.at("/xmlns:registry")
+        doc = Parser.parse registry, nil, @errors
+        save_doc doc
+        registry.xpath("./xmlns:registry").each { |r| save_doc Parser.parse(r, doc, @errors) }
+      end
+
+      #
+      # Save document to file
+      #
+      # @param [RelatonIana::IanaBibliographicItem, nil] bib bibliographic item
+      #
+      def save_doc(bib) # rubocop:disable Metrics/MethodLength
+        return unless bib
+
+        # Two distinct registries can sanitize to one filename (`/` and `-` both
+        # collapse to `-`), e.g. `rpki/signed-objects` and `rpki-signed-objects`.
+        # Take a path of our own rather than overwriting the other document, but
+        # keep the clash visible in the crawl log.
+        file = unique_output_file bib.docnumber
+        if @files.include? file
+          # A reserved path only ever belongs to one docnumber, so this is the
+          # same document again: a genuine duplicate, not a collision. Checked
+          # FIRST, because a disambiguated path stays != output_file forever.
+          Util.warn "File #{file} already exists. Document: #{bib.docnumber}"
+        elsif file != output_file(bib.docnumber)
+          Util.warn "File #{output_file bib.docnumber} already exists. " \
+                    "Document: #{bib.docnumber}. Writing #{file} instead."
+        end
+        @files << file
+        add_to_index bib.docnumber, file
+        File.write file, serialize(bib), encoding: "UTF-8"
+      end
+
+      def to_xml(bib)
+        bib.to_xml(bibdata: true)
+      end
+
+      def to_yaml(bib)
+        bib.to_yaml
+      end
+
+      def to_bibxml(bib)
+        bib.to_rfcxml
+      end
+    end
+  end
+end

data/lib/relaton/iana/item.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module Iana
+    class Item < Bib::Item
+      model ItemData
+    end
+  end
+end

data/lib/relaton/iana/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module Iana
+    class ItemData < Bib::ItemData
+    end
+  end
+end