relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/ietf/rfc/index.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "rfc_index_namespace"
+require_relative "entry"
+
+module Relaton
+  module Ietf
+    module Rfc
+      # Model for the root <rfc-index> element
+      class Index < Lutaml::Model::Serializable
+        attribute :bcp_entries, Entry, collection: true
+        attribute :fyi_entries, Entry, collection: true
+        attribute :std_entries, Entry, collection: true
+        attribute :rfc_entries, Entry, collection: true
+
+        xml do
+          root "rfc-index"
+          namespace RfcIndexNamespace
+
+          map_element "bcp-entry", to: :bcp_entries
+          map_element "fyi-entry", to: :fyi_entries
+          map_element "std-entry", to: :std_entries
+          map_element "rfc-entry", to: :rfc_entries
+        end
+
+        #
+        # Get all subseries entries
+        #
+        # @return [Array<Entry>]
+        #
+        def subseries_entries
+          (bcp_entries || []) + (fyi_entries || []) + (std_entries || [])
+        end
+
+        #
+        # Get entries that have is-also references (can be converted to items)
+        #
+        # @return [Array<Entry>]
+        #
+        def parseable_entries
+          subseries_entries.select(&:has_is_also?)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ietf/rfc/is_also.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "rfc_index_namespace"
+
+module Relaton
+  module Ietf
+    module Rfc
+      # Model for <is-also> element containing doc-id references
+      class IsAlso < Lutaml::Model::Serializable
+        attribute :doc_id, :string, collection: true
+
+        xml do
+          root "is-also"
+          namespace RfcIndexNamespace
+
+          map_element "doc-id", to: :doc_id
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ietf/rfc/keywords.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "rfc_index_namespace"
+
+module Relaton
+  module Ietf
+    module Rfc
+      class Keywords < Lutaml::Model::Serializable
+        attribute :kw, :string, collection: true
+
+        xml do
+          root "keywords"
+          namespace RfcIndexNamespace
+          map_element "kw", to: :kw
+        end
+      end
+    end
+  end
+end

data/lib/relaton/ietf/rfc/rfc_index_namespace.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Relaton
+  module Ietf
+    module Rfc
+      class RfcIndexNamespace < Lutaml::Xml::W3c::XmlNamespace
+        uri "https://www.rfc-editor.org/rfc-index"
+      end
+    end
+  end
+end

data/lib/relaton/ietf/scraper.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "pubid"
+require "pubid/ietf"
+
+module Relaton
+  module Ietf
+    # Scraper module
+    module Scraper
+      extend Scraper
+
+      # The combined corpus — RFCs, the RFC sub-series and Internet-Drafts in one
+      # repo, with one pubid `index-v2` covering all ~177k records (relaton#109).
+      # It replaces the three per-type repos this flavor used to read; those keep
+      # publishing their `index-v1` for released relatons, untouched.
+      IETF = "https://raw.githubusercontent.com/relaton/relaton-data-ietf/main/"
+
+      # @param text [String]
+      # @return [Relaton::Ietf::ItemData, nil]
+      def scrape_page(text)
+        id = parse_id text.sub(/\AIETF\s+/, "")
+        return unless id
+
+        fetch_doc id
+      rescue Timeout::Error, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+             Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+             Net::ProtocolError, SocketError
+        raise Relaton::RequestError, "No document found for #{text} reference"
+      end
+
+      private
+
+      # The index stores parsed pubids, so the query has to be one too.
+      # `Type#search_candidates` narrows only when the query is not a String
+      # (`@file_io.sorted && id && !id.is_a?(String)`); a String falls through to
+      # `match_item`'s `item[:id].to_s.include?(id)`, which renders every pubid in
+      # the index on every lookup — measured at ~40 s per reference against the
+      # 177k-row index, versus sub-millisecond for a parsed one.
+      # `exact:` keeps the match exact. `Type#search` without it takes
+      # pubid's subset match, where a component the query omits is a wildcard —
+      # and an Internet-Draft slug omits the version, so `draft-foo` would match
+      # every `draft-foo-NN` and `.first` would return an arbitrary one (20,513
+      # such pairs in the first 40k rows of the index fixture).
+      def fetch_doc(id)
+        row = index.search(id, exact: true).first
+        get_page "#{IETF}#{row[:file]}" if row
+      end
+
+      def index
+        Relaton::Index.find_or_create(
+          :IETF, url: "#{IETF}#{INDEXFILE}.zip", file: "#{INDEXFILE}.yaml",
+                 pubid_class: ::Pubid::Ietf::Identifier
+        )
+      end
+
+      #
+      # Parse a reference into the identifier the index is keyed by.
+      #
+      # Normalises the two Internet-Draft spellings callers use — `I-D.<slug>`
+      # and `I-D <slug>` — onto the `draft-…` form pubid parses and the index
+      # stores. The bare `I-D.ietf-quic-transport` spelling (the bibxml anchor,
+      # and the `docnumber` IETF records carry) gains the `draft-` stem: the old
+      # plain-string index matched it by substring, and matching is exact now.
+      #
+      # The capture is `(\S.*)`, not `(.+)`: it must start non-space so it cannot
+      # overlap the preceding `\s*`. Behaviour is unchanged on every real
+      # reference — with `(.+)` the greedy `\s*` only ever yields ground on an
+      # all-whitespace remainder, which pubid rejects anyway — but the
+      # unambiguous form is about twice as fast on a pathological input and is
+      # what CodeQL's rb/polynomial-redos models. Measured before changing it:
+      # the old form was already linear (0.66 ms at 80k chars), so this is
+      # clarity, not a ReDoS fix.
+      #
+      # @param ref [String]
+      # @return [Pubid::Ietf::Identifier, nil] nil when pubid has no grammar for
+      #   it, so an out-of-flavor reference logs "Not found." instead of raising
+      #
+      def parse_id(ref)
+        if (draft = ref[/\AI-D[.\s]\s*(\S.*)\z/m, 1])
+          ref = draft.start_with?("draft-") ? draft : "draft-#{draft}"
+        end
+        ::Pubid::Ietf::Identifier.parse ref
+      rescue StandardError => e
+        # Logged, not swallowed: this repo git-pins pubid to a moving `main`,
+        # so a grammar regression would otherwise present as every IETF
+        # reference quietly reporting "Not found."
+        Util.debug "`#{ref}` is not an IETF identifier: #{e.message}"
+        nil
+      end
+
+      # @param uri [String]
+      # @return [Relaton::Ietf::ItemData, nil] HTTP response body
+      def get_page(uri)
+        res = Net::HTTP.get_response(URI(uri))
+        return unless res.code == "200"
+
+        Item.from_yaml(res.body).tap { |item| item.fetched = Date.today.to_s }
+      end
+    end
+  end
+end

data/lib/relaton/ietf/util.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Ietf
+    module Util
+      extend Relaton::Bib::Util
+      PROGNAME = "relaton-ietf".freeze
+    end
+  end
+end

data/lib/relaton/ietf/wg_name_resolver.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+
+module Relaton
+  module Ietf
+    # Fetches working group and research group names from the IETF Datatracker API.
+    # Returns a Hash mapping acronym (lowercase) to full group name.
+    module WgNameResolver
+      API_URL = "https://datatracker.ietf.org/api/v1/group/group/"
+
+      # @return [Hash{String => String}] acronym => full name
+      def self.fetch
+        result = {}
+        offset = 0
+        limit = 1000
+        loop do
+          uri = URI("#{API_URL}?type__in=wg,rg&limit=#{limit}&offset=#{offset}&format=json")
+          response = Net::HTTP.get_response(uri)
+          break unless response.is_a?(Net::HTTPSuccess)
+
+          data = JSON.parse(response.body)
+          objects = data["objects"] || []
+          break if objects.empty?
+
+          objects.each do |group|
+            acronym = group["acronym"]
+            name = group["name"]
+            result[acronym] = name if acronym && name
+          end
+          offset += limit
+          break unless data.dig("meta", "next")
+        end
+        result
+      rescue StandardError => e
+        Util.warn "Failed to fetch WG names from Datatracker: #{e.message}"
+        {}
+      end
+    end
+  end
+end

data/lib/relaton/ietf.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "relaton/index"
+require "relaton/bib"
+require_relative "version"
+require_relative "ietf/util"
+require_relative "ietf/item_data"
+require_relative "ietf/item"
+require_relative "ietf/bibitem"
+require_relative "ietf/bibdata"
+require_relative "ietf/rfc/index"
+require_relative "ietf/bibliography"
+
+module Relaton
+  module Ietf
+    # The pubid-structured index, written by DataFetcher and read by Scraper
+    # (relaton#109). One constant again now that both sides are on it — the
+    # second one existed only while producer and consumer were split.
+    INDEXFILE = "index-v2".freeze
+    # Returns hash of XML reammar
+    # @return [String]
+    def self.grammar_hash
+      # gem_path = File.expand_path "..", __dir__
+      # grammars_path = File.join gem_path, "grammars", "*"
+      # grammars = Dir[grammars_path].sort.map { |gp| File.read gp }.join
+      Digest::MD5.hexdigest Relaton::VERSION # grammars
+    end
+  end
+end

data/lib/relaton/iho/bibdata.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iho
+    class Bibdata < Item
+      model ItemData
+      include Bib::BibdataShared
+    end
+  end
+end

data/lib/relaton/iho/bibitem.rb

@@ -0,0 +1,8 @@
+module Relaton
+  module Iho
+    class Bibitem < Item
+      model ItemData
+      include Bib::BibitemShared
+    end
+  end
+end

data/lib/relaton/iho/bibliography.rb

@@ -0,0 +1,114 @@
+require "net/http"
+
+module Relaton
+  module Iho
+    module Bibliography
+      ENDPOINT = "https://raw.githubusercontent.com/relaton/relaton-data-iho/refs/heads/v2/".freeze
+
+      class << self
+        #
+        # Search for IHO standard by IHO standard Code
+        #
+        # @param text [String] the IHO standard Code to look up (e..g "IHO B-11")
+        #
+        # @return [RelatonIho::IhoBibliographicItem, nil] the IHO standard or nil if not found
+        #
+        def search(text, _year = nil, _opts = {}) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+          pubid = text.is_a?(String) ? ::Pubid::Iho::Identifier.parse(text) : text
+          Util.info "Fetching from Relaton repository ...", key: pubid.to_s
+          # Pass the pubid so Relaton::Index narrows candidates by number via
+          # binary search before applying the block. Every row's `:id` is a
+          # Pubid::Iho::Identifiers::* (Relaton::Index deserialized it via the
+          # `pubid_class` passed in `#index`), so `pubid_match?` compares pubids.
+          row = index.search(pubid) { |r| pubid_match?(r[:id], pubid) }
+                     .min_by { |r| r[:id].version.to_s }
+          unless row
+            Util.info "Not found.", key: pubid.to_s
+            return
+          end
+
+          uri = URI("#{ENDPOINT}#{row[:file]}")
+          resp = Net::HTTP.get_response uri
+          unless resp.code == "200"
+            raise Relaton::RequestError, "Could not access #{uri}: HTTP #{resp.code}"
+          end
+
+          item = Relaton::Iho::Item.from_yaml resp.body
+          enrich_with_pubid(item, pubid)
+          Util.info "Found: `#{item.docidentifier.first.content}`", key: pubid.to_s
+          item.tap { |i| i.fetched = Date.today.to_s }
+        rescue SocketError, Errno::EINVAL, Errno::ECONNRESET, EOFError,
+              Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError,
+              Net::ProtocolError, Net::ReadTimeout, OpenSSL::SSL::SSLError,
+              Errno::ETIMEDOUT => e
+          raise Relaton::RequestError, "Could not access #{uri}: #{e.message}"
+        end
+
+        # Populate ext.structuredidentifier from the parsed Pubid when the
+        # fetched record doesn't already provide one. Maps `pubid.number`
+        # (with type prefix, e.g. `S-100`) -> docnumber, `pubid.part` -> part,
+        # `pubid.appendix` -> appendixid, `pubid.annex` -> annexid,
+        # `pubid.supplement` -> supplementid.
+        def enrich_with_pubid(item, pubid)
+          return if item.ext&.structuredidentifier&.any?
+
+          sid = StructuredIdentifier.new(
+            docnumber: pubid_docnumber(pubid),
+            part: pubid.part,
+            appendixid: (pubid.appendix if pubid.respond_to?(:appendix)),
+            annexid: (pubid.annex if pubid.respond_to?(:annex)),
+            supplementid: (pubid.supplement if pubid.respond_to?(:supplement)),
+          )
+          item.ext ||= Ext.new
+          item.ext.structuredidentifier = [sid]
+        end
+
+        # "S-100" rather than "100" — keeps the type prefix that distinguishes
+        # the IHO series (S/P/M/B/C). Matches spec/fixtures/iho_part.xml.
+        def pubid_docnumber(pubid)
+          pubid.to_s.sub(/^IHO\s/, "").split(/\s+/, 2).first
+        end
+
+        # @param ref [String] the IHO standard Code to look up (e..g "IHO B-11")
+        # @param year [String] the year the standard was published (optional)
+        #
+        # @param opts [Hash] options
+        # @option opts [TrueClass, FalseClass] :all_parts restricted to all parts
+        #   if all-parts reference is required
+        # @option opts [TrueClass, FalseClass] :bibdata
+        #
+        # @return [RelatonIho::IhoBibliographicItem]
+        def get(ref, year = nil, opts = {})
+          search(ref, year, opts)
+        end
+
+        private
+
+        def index
+          Relaton::Index.find_or_create(
+            :iho,
+            url: "#{ENDPOINT}#{INDEXFILE}.zip",
+            file: "#{INDEXFILE}.yaml",
+            pubid_class: ::Pubid::Iho::Identifier,
+          )
+        end
+
+        # Both `row_id` and `query` are Pubid::Iho::Identifiers::* instances.
+        # Class identity pins the series (S/P/M/B/C). Subdivision keys
+        # (part/appendix/annex/supplement) use strict equality — a nil query
+        # must match a nil row (the umbrella), not an arbitrary subdivision
+        # under the same (number, version). Only :version stays nil-tolerant: an
+        # unqualified `IHO B-11` query is expected to find the latest edition.
+        def pubid_match?(row_id, query)
+          row_id.class == query.class &&
+            row_id.number == query.number &&
+            row_id.part.to_s == query.part.to_s &&
+            row_id.appendix.to_s == query.appendix.to_s &&
+            row_id.annex.to_s == query.annex.to_s &&
+            row_id.supplement.to_s == query.supplement.to_s &&
+            (query.version.nil? || row_id.version.to_s == query.version.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/relaton/iho/comment_period.rb

@@ -0,0 +1,13 @@
+module Relaton
+  module Iho
+    class CommentPeriod < Lutaml::Model::Serializable
+      attribute :from, :date
+      attribute :to, :date
+
+      xml do
+        map_element "from", to: :from
+        map_element "to", to: :to
+      end
+    end
+  end
+end

data/lib/relaton/iho/docidentifier.rb

@@ -0,0 +1,31 @@
+module Relaton
+  module Iho
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def initialize(attrs = {}, options = {})
+        pubid = attrs.is_a?(Hash) ? attrs.delete(:pubid) : nil
+        attrs[:content] ||= pubid.to_s if pubid
+        super
+        @pubid = pubid if pubid
+      end
+
+      def content=(value)
+        super
+        @pubid = ::Pubid::Iho::Identifier.parse(value) if value
+      end
+
+      def remove_part!
+        @pubid&.part = nil
+      end
+
+      def remove_date!
+        @pubid&.date = nil
+      end
+
+      def to_all_parts!
+        @pubid &&= @pubid.to_all_parts
+      end
+    end
+  end
+end

data/lib/relaton/iho/doctype.rb

@@ -0,0 +1,10 @@
+module Relaton
+  module Iho
+    class Doctype < Bib::Doctype
+      TYPES = %w[policy-and-procedures best-practices supporting-document
+                 report legal directives proposal standard].freeze
+
+      attribute :type, :string, values: TYPES
+    end
+  end
+end

data/lib/relaton/iho/ext.rb

@@ -0,0 +1,20 @@
+require_relative "doctype"
+require_relative "comment_period"
+require_relative "structured_identifier"
+
+module Relaton
+  module Iho
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype, default: -> { Doctype.new(content: "standard") }
+      attribute :commentperiod, CommentPeriod
+      attribute :structuredidentifier, StructuredIdentifier,
+                collection: true, initialize_empty: true
+
+      xml { map_element "commentperiod", to: :commentperiod }
+
+      key_value { map_element "commentperiod", to: :commentperiod }
+
+      def schema_version = Relaton.schema_versions["relaton-model-iho"]
+    end
+  end
+end

data/lib/relaton/iho/hash_parser_v1.rb

@@ -0,0 +1,144 @@
+require "relaton/bib/hash_parser_v1"
+require_relative "../iho"
+
+module Relaton
+  module Iho
+    module HashParserV1
+      include Core::ArrayWrapper
+      include Core::DateParser
+      include Bib::HashParserV1
+      extend self
+
+      private
+
+      def ext_hash_to_bib(ret) # rubocop:disable Metrics/AbcSize
+        ret[:ext] ||= {}
+        doctype_hash_to_bib ret
+        ret[:ext][:doctype] ||= Doctype.new(content: "standard")
+        ret[:ext][:subdoctype] = ret.delete(:subdoctype) if ret[:subdoctype]
+        ret[:ext][:flavor] ||= flavor(ret)
+        ics_hash_to_bib ret
+        commentperiod_hash_to_bib ret
+        structuredidentifier_hash_to_bib ret
+        ret[:ext] = Ext.new(**ret[:ext])
+      end
+
+      def structuredidentifier_hash_to_bib(ret)
+        sid = ret.dig(:ext, :structuredidentifier) || ret[:structuredidentifier]
+        return unless sid
+
+        ret[:ext]&.delete(:structuredidentifier)
+        ret.delete(:structuredidentifier)
+        ret[:ext][:structuredidentifier] =
+          array(sid).map { |s| StructuredIdentifier.new(**s) }
+      end
+
+      def commentperiod_hash_to_bib(ret)
+        cp = ret.dig(:ext, :commentperiod) || ret[:commentperiod]
+        return unless cp
+
+        ret[:ext]&.delete(:commentperiod)
+        ret.delete(:commentperiod)
+        cp[:from] = ::Date.parse(cp[:from]) if cp[:from]
+        cp[:to] = ::Date.parse(cp[:to]) if cp[:to]
+        ret[:ext][:commentperiod] = CommentPeriod.new(**cp)
+      end
+
+      # IHO editorialgroup is an array of arrays of committee entries:
+      #   editorialgroup:
+      #   - - committee:
+      #         abbreviation: HSSC
+      #         name: Hydrographic Services and Standards Committee
+      #     - committee:
+      #         abbreviation: IRCC
+      #         name: Inter-Regional Coordination Committee
+      #         committee:
+      #           abbreviation: GEBCO
+      #           name: JOINT IHO-IOC GUIDING COMMITTEE FOR THE GENERAL BATHYMETRIC CHART OF THE OCEANS
+      def editorialgroup_hash_to_bib(ret) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+        eg = ret.dig(:ext, :editorialgroup) || ret[:editorialgroup]
+        return unless eg
+
+        ret[:ext]&.delete(:editorialgroup)
+        ret.delete(:editorialgroup)
+        ret[:contributor] ||= []
+
+        array(eg).flatten.each do |entry|
+          committee = entry[:committee]
+          next unless committee
+
+          subdivision = build_subdivision(committee)
+          ret[:contributor] << Bib::Contributor.new(
+            role: [Bib::Contributor::Role.new(
+              type: "author",
+              description: [Bib::LocalizedMarkedUpString.new(content: "committee")],
+            )],
+            organization: Bib::Organization.new(
+              name: [Bib::TypedLocalizedString.new(content: "International Hydrographic Organization")],
+              abbreviation: Bib::LocalizedString.new(content: "IHO"),
+              subdivision: [subdivision],
+            ),
+          )
+        end
+      end
+
+      def build_subdivision(committee)
+        sub_divisions = []
+        if committee[:committee]
+          sub_divisions << build_subdivision(committee[:committee])
+        end
+
+        identifiers = []
+        if committee[:abbreviation]
+          identifiers << Bib::OrganizationType::Identifier.new(
+            content: committee[:abbreviation],
+          )
+        end
+
+        Bib::Subdivision.new(
+          type: "technical-committee",
+          name: [Bib::TypedLocalizedString.new(content: committee[:name])],
+          identifier: identifiers,
+          subdivision: sub_divisions,
+        )
+      end
+
+      def series_hash_to_bib(ret)
+        ret[:series] &&= array(ret[:series]).map do |s|
+          if s[:title].is_a?(Hash) && s[:title][:content].is_a?(Array)
+            s[:title] = s[:title][:content].map do |c|
+              { type: s[:title][:type], content: c[:content], language: c[:language], script: c[:script] }.compact
+            end
+          end
+          super_series(s)
+        end
+      end
+
+      def super_series(s)
+        s[:formattedref] && s[:formattedref] = formattedref(s[:formattedref])
+        s[:title] &&= title_collection(s[:title])
+        s[:place] &&= create_place(s[:place])
+        s[:abbreviation] &&= localizedstring(s[:abbreviation])
+        s[:from] &&= ::Date.parse(s[:from])
+        s[:to] &&= ::Date.parse(s[:to])
+        Bib::Series.new(**s)
+      end
+
+      def bib_item(item)
+        ItemData.new(**item)
+      end
+
+      def create_doctype(args)
+        Doctype.new(**args)
+      end
+
+      def create_relation(rel)
+        Relation.new(**rel)
+      end
+
+      def create_docid(**args)
+        Docidentifier.new(**args)
+      end
+    end
+  end
+end

data/lib/relaton/iho/item.rb

@@ -0,0 +1,20 @@
+require_relative "item_data"
+require_relative "ext"
+require_relative "docidentifier"
+
+module Relaton
+  module Iho
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end
+
+require_relative "relation"
+
+Relaton::Iho::Item.attribute :relation, Relaton::Iho::Relation,
+                             collection: true, initialize_empty: true

data/lib/relaton/iho/item_base.rb

@@ -0,0 +1,18 @@
+module Relaton
+  module Iho
+    class ItemBase < Lutaml::Model::Serializable
+      include Bib::NamespaceHelper
+
+      attr_accessor :type
+
+      model ItemData
+
+      instance_exec(&Bib::ItemShared::ATTRIBUTES)
+
+      xml do
+        map_attribute "type", to: :type
+        instance_exec(&Bib::ItemShared::XML_BODY)
+      end
+    end
+  end
+end