relaton 2.2.0.pre.alpha.1 → 3.0.0.pre.alpha.2

data/lib/relaton/w3c/data_parser.rb

@@ -0,0 +1,379 @@
+module Relaton
+  module W3c
+    class DataParser
+      include Relaton::W3c::SafeRealize
+
+      USED_TYPES = %w[WD NOTE PER PR REC CR].freeze
+
+      DOCTYPES = {
+        "TR" => "technicalReport",
+        "NOTE" => "groupNote",
+      }.freeze
+
+      STAGES = {
+        "RET" => "Retired",
+        "SPSD" => "Superseded Recommendation",
+        "OBSL" => "Obsoleted Recommendation",
+        "WD" => "Working Draft",
+        "CRD" => "Candidate Recommendation Draft",
+        "CR" => "Candidate Recommendation",
+        "PR" => "Proposed Recommendation",
+        "PER" => "Proposed Edited Recommendation",
+        "REC" => "Recommendation",
+      }.freeze
+
+      #
+      # Document parser initalization
+      #
+      # @param [W3cApi::Models::SpecVersion] spec
+      #
+      ERROR_KEYS = %i[status title doc_uri formattedref series date
+                      relation contributor doctype].freeze
+
+      def initialize(spec, errors = {})
+        @spec = spec
+        @errors = errors
+        ERROR_KEYS.each { |k| @errors[k] = true unless @errors.key?(k) }
+      end
+
+      #
+      # Initialize document parser and run it
+      #
+      # @param [W3cApi::Models::SpecVersion] spec
+      #
+      # @return [Relaton::W3c::ItemData, nil] bibliographic item
+      #
+      def self.parse(spec, errors = {})
+        new(spec, errors).parse
+      end
+
+      #
+      # Parse document
+      #
+      # @return [Relaton::W3c::ItemData] bibliographic item
+      #
+      def parse # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        ItemData.new(
+          type: "standard",
+          language: ["en"],
+          script: ["Latn"],
+          status: parse_status,
+          title: parse_title,
+          source: parse_source,
+          docidentifier: parse_docid,
+          formattedref: parse_formattedref,
+          docnumber: identifier,
+          series: parse_series,
+          date: parse_date,
+          relation: parse_relation,
+          contributor: parse_contrib,
+          ext: parse_ext,
+        )
+      end
+
+      #
+      # Parse ext with doctype
+      #
+      # @return [Ext, nil] ext
+      #
+      def parse_ext
+        dt = parse_doctype
+        result = Ext.new(doctype: dt, flavor: "w3c")
+        result
+      end
+
+      #
+      # Extract document status
+      #
+      # @return [Bib::Status, nil] document status
+      #
+      def parse_status
+        result = if @spec.respond_to?(:status) && @spec.status
+                   Bib::Status.new(stage: Bib::Status::Stage.new(content: @spec.status))
+                 end
+        @errors[:status] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse title
+      #
+      # @return [Array<Bib::Title>] title
+      #
+      def parse_title(spec = @spec)
+        return [] unless spec&.title && spec.title.strip != ""
+
+        result = [Bib::Title.new(content: spec.title, language: "en", script: "Latn")]
+        @errors[:title] &&= result.empty?
+        result
+      end
+
+      def doc_uri(spec = @spec)
+        result = spec.respond_to?(:uri) ? spec.uri : spec.shortlink
+        @errors[:doc_uri] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse link
+      #
+      # @return [Array<Bib::Uri>] link
+      #
+      def parse_source
+        [Bib::Uri.new(type: "src", content: doc_uri)]
+      end
+
+      #
+      # Parse docidentifier
+      #
+      # @return [Array<Docidentifier>] docidentifier
+      #
+      def parse_docid
+        id = pub_id(doc_uri)
+        [Docidentifier.new(type: "W3C", content: id, primary: true)]
+      end
+
+      #
+      # Generate PubID
+      #
+      # @return [String] PubID
+      #
+      def pub_id(url)
+        "W3C #{identifier(url)}"
+      end
+
+      #
+      # Generate identifier from URL
+      #
+      # @param [String] link
+      #
+      # @return [String] identifier
+      #
+      def identifier(link = doc_uri)
+        self.class.parse_identifier(link)
+      end
+
+      #
+      # Parse identifier from URL
+      #
+      # @param [String] url URL
+      #
+      # @return [String] identifier
+      #
+      def self.parse_identifier(url)
+        if /.+\/(\w+(?:[-+][\w.]+)+(?:\/\w+)?)/ =~ url.to_s
+          $1.to_s
+        else url.to_s.split("/").last
+        end
+      end
+
+      #
+      # Parse series
+      #
+      # @return [Array<Bib::Series>] series
+      #
+      def parse_series
+        result = if type
+                   title = Bib::Title.new(content: "W3C #{type}", language: "en", script: "Latn")
+                   [Bib::Series.new(title: [title], number: identifier)]
+                 else
+                   []
+                 end
+        @errors[:series] &&= result.empty?
+        result
+      end
+
+      #
+      # Extract type
+      #
+      # @return [String] type
+      #
+      def type
+        @type ||= @spec.respond_to?(:status) ? @spec.status : "technicalReport"
+      end
+
+      #
+      # Parse doctype
+      #
+      # @return [Doctype, nil] doctype
+      #
+      def parse_doctype
+        t = DOCTYPES[type] || DOCTYPES[type_from_link]
+        result = Doctype.new(content: t) if t
+        @errors[:doctype] &&= result.nil?
+        result
+      end
+
+      #
+      # Fetch type from link
+      #
+      # @return [String, nil] type
+      #
+      def type_from_link
+        @spec.shortlink.strip.match(/www\.w3\.org\/(TR)/)&.to_a&.fetch 1
+      end
+
+      #
+      # Parse date
+      #
+      # @return [Array<Bib::Date>] date
+      #
+      def parse_date
+        result = if @spec.respond_to?(:date)
+                   [Bib::Date.new(type: "published", at: @spec.date.to_date.to_s)]
+                 else
+                   []
+                 end
+        @errors[:date] &&= result.empty?
+        result
+      end
+
+      #
+      # Parse relation
+      #
+      # @return [Array<Bib::Relation>] relation
+      #
+      def parse_relation
+        result = if @spec.links.respond_to?(:version_history)
+                   version_history = realize @spec.links.version_history
+                   version_history&.links&.spec_versions&.map { |version| create_relation(version, "hasEdition") } || []
+                 else
+                   relations
+                 end
+        result = result.compact
+        @errors[:relation] &&= result.empty?
+        result
+      end
+
+      #
+      # Create relations
+      #
+      # @return [Array<Bib::Relation>] relations
+      #
+      def relations # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+        rels = []
+        rels << create_relation(@spec.links.specification, "editionOf") if @spec.links.respond_to?(:specification)
+        if @spec.links.respond_to?(:predecessor_versions) && @spec.links.predecessor_versions
+          predecessor_versions = realize @spec.links.predecessor_versions
+          predecessor_versions&.links&.predecessor_versions&.each do |version|
+            rels << create_relation(version, "obsoletes")
+          end
+        end
+        if @spec.links.respond_to?(:successor_versions) && @spec.links.successor_versions
+          successor_versions = realize @spec.links.successor_versions
+          successor_versions&.links&.successor_versions&.each do |version|
+            rels << create_relation(version, "updatedBy", "errata")
+          end
+        end
+        rels.compact
+      end
+
+      #
+      # Create relation
+      #
+      # @param [Object] version version link
+      # @param [String] type relation type
+      # @param [String, nil] desc relation description
+      #
+      # @return [Bib::Relation] relation
+      #
+      def create_relation(version, type, desc = nil)
+        version_spec = realize version
+        return nil unless version_spec
+
+        url = doc_uri(version_spec)
+        id = pub_id(url)
+        title = parse_title(version_spec)
+        docid = Bib::Docidentifier.new(type: "W3C", content: id, primary: true)
+        link = [Bib::Uri.new(type: "src", content: url)]
+        bib = ItemData.new(title: title, docidentifier: [docid], source: link)
+        dsc = Bib::LocalizedMarkedUpString.new(content: desc) if desc
+        Bib::Relation.new(type: type, bibitem: bib, description: dsc)
+      end
+
+      #
+      # Parse formattedref
+      #
+      # @return [String, nil] formattedref
+      #
+      def parse_formattedref
+        result = if @spec.respond_to?(:uri)
+                   Bib::Formattedref.new(content: pub_id(@spec.uri))
+                 end
+        @errors[:formattedref] &&= result.nil?
+        result
+      end
+
+      #
+      # Parse contributor
+      #
+      # @return [Array<Bib::Contributor>] contributor
+      #
+      def parse_contrib # rubocop:disable Metrics/MethodLength
+        contribs = [Bib::Contributor.new(
+          organization: create_w3c_org,
+          role: [Bib::Contributor::Role.new(type: "publisher")],
+        )]
+
+        if @spec.links.respond_to?(:editors)
+          editors = realize @spec.links.editors
+          editors&.links&.editors&.each do |ed|
+            editor = create_editor(ed)
+            contribs << editor if editor
+          end
+        end
+
+        result = contribs + parse_deliverers
+        @errors[:contributor] &&= result.empty?
+        result
+      end
+
+      def create_editor(unrealized_editor)
+        editor = realize unrealized_editor
+        return unless editor
+
+        surname = Bib::LocalizedString.new(content: editor.family, language: "en", script: "Latn")
+        forename = Bib::FullNameType::Forename.new(content: editor.given, language: "en", script: "Latn")
+        name = Bib::FullName.new(surname: surname, forename: [forename])
+        person = Bib::Person.new(name: name)
+        Bib::Contributor.new(
+          person: person,
+          role: [Bib::Contributor::Role.new(type: "editor")],
+        )
+      end
+
+      #
+      # Parse deliverers as contributors with role "author" and description "committee"
+      #
+      # @return [Array<Bib::Contributor>] deliverer contributors
+      #
+      def parse_deliverers # rubocop:disable Metrics/MethodLength
+        return [] unless @spec.links.respond_to?(:deliverers)
+
+        deliverers = realize @spec.links.deliverers
+        return [] unless deliverers&.links&.deliverers
+
+        deliverers.links.deliverers.map do |edg|
+          org = create_w3c_org.tap do |o|
+            subdiv_name = Bib::TypedLocalizedString.new(content: edg.title)
+            subdiv = Bib::Subdivision.new(name: [subdiv_name], type: "technical-committee")
+            o.subdivision = [subdiv]
+          end
+          role = Bib::Contributor::Role.new(
+            type: "author",
+            description: [Bib::LocalizedMarkedUpString.new(content: "committee")],
+          )
+          Bib::Contributor.new(organization: org, role: [role])
+        end
+      end
+
+      def create_w3c_org
+        Bib::Organization.new(
+          name: [Bib::TypedLocalizedString.new(content: "World Wide Web Consortium")],
+          abbreviation: Bib::LocalizedString.new(content: "W3C"),
+          uri: [Bib::Uri.new(content: "https://www.w3.org")],
+        )
+      end
+    end
+  end
+end

data/lib/relaton/w3c/docidentifier.rb

@@ -0,0 +1,48 @@
+module Relaton
+  module W3c
+    # A W3C document identifier that carries its parsed pubid alongside the
+    # source string.
+    #
+    # `DataParser#pub_id` builds `"W3C REC-xml-names-20091208"` — publisher
+    # prefix included — which is exactly the form
+    # `Pubid::W3c::Identifier.parse` accepts, so nothing has to synthesize or
+    # strip a prefix. `DataFetcher#index_primary` takes `#pubid` straight from
+    # here to build the `index-v2` rows.
+    #
+    # Follows the IHO shape rather than the ISO one: `content=` calls `super`
+    # first, so `content` keeps the source string verbatim and serialization is
+    # unchanged. ISO re-renders `content` from the pubid, which W3C has no need
+    # of and which would alter its published output.
+    class Docidentifier < Bib::Docidentifier
+      attr_reader :pubid
+
+      def content=(value)
+        super
+        @pubid = value && parse(value)
+      end
+
+      # W3C identifiers carry no part, date-as-component or all-parts notion, so
+      # the three mutators `Bib::ItemData` broadcasts over every docidentifier
+      # are no-ops here rather than the inherited NotImplementedError.
+      def remove_part!; end
+
+      def remove_date!; end
+
+      def to_all_parts!; end
+
+      private
+
+      # An identifier that does not parse is a data defect, so it is reported at
+      # ERROR — never at WARN. It does not raise: an already-published record
+      # still has to deserialize and render. The crawl escalates the same
+      # failure into a tracked GitHub issue (see
+      # `DataFetcher#index_primary` and `Core::DataFetcher#report_errors`).
+      def parse(value)
+        ::Pubid::W3c::Identifier.parse value.to_s
+      rescue StandardError => e
+        Util.error "Failed to parse pubid `#{value}`: #{e.message}"
+        nil
+      end
+    end
+  end
+end

data/lib/relaton/w3c/doctype.rb

@@ -0,0 +1,7 @@
+module Relaton
+  module W3c
+    class Doctype < Bib::Doctype
+      attribute :content, :string, values: %w[groupNote technicalReport]
+    end
+  end
+end

data/lib/relaton/w3c/ext.rb

@@ -0,0 +1,11 @@
+require_relative "doctype"
+
+module Relaton
+  module W3c
+    class Ext < Bib::Ext
+      attribute :doctype, Doctype
+
+      def get_schema_version = Relaton.schema_versions["relaton-model-w3c"]
+    end
+  end
+end

data/lib/relaton/w3c/governor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "lutaml/hal"
+require_relative "../core/governor"
+
+module Relaton
+  module W3c
+    #
+    # W3C binding of {Relaton::Core::Governor}.
+    #
+    # The ladder (60 s → 900 s), the per-round escalation, the jitter, the
+    # latched give-up and the Retry-After parsing all live in core, where the
+    # ITU-R crawler shares them. What is W3C-specific is exactly two things:
+    # which errors mean "rate-limited", and which env vars tune the durations.
+    #
+    class Governor < Relaton::Core::Governor
+      # Errors that mean "you are being rate-limited", as opposed to "this
+      # resource is broken". api.w3.org signals throttling with **403 as well as
+      # 429** — w3c_api's whole faraday-retry layer is built around 403 being the
+      # W3C rate-limit signal — so a 403 must never be recorded as a permanently
+      # broken resource either. Requires lutaml-hal >= 0.2.5, which gave 403 its
+      # own ForbiddenError class (before that it arrived as a bare Error).
+      THROTTLE_ERRORS = [
+        ::Lutaml::Hal::TooManyRequestsError,
+        ::Lutaml::Hal::ForbiddenError,
+      ].freeze
+
+      # RELATON_W3C_THROTTLE_BASE / _MAX / _GIVEUP.
+      ENV_PREFIX = "RELATON_W3C"
+    end
+  end
+end

data/lib/relaton/w3c/item.rb

@@ -0,0 +1,15 @@
+require_relative "item_data"
+require_relative "ext"
+require_relative "docidentifier"
+
+module Relaton
+  module W3c
+    class Item < Bib::Item
+      model ItemData
+
+      attribute :ext, Ext
+      attribute :docidentifier, Docidentifier, collection: true,
+                                               initialize_empty: true
+    end
+  end
+end

data/lib/relaton/w3c/item_data.rb

@@ -0,0 +1,6 @@
+module Relaton
+  module W3c
+    class ItemData < Bib::ItemData
+    end
+  end
+end

data/lib/relaton/w3c/processor.rb

@@ -0,0 +1,49 @@
+require "relaton/core/processor"
+
+module Relaton
+  module W3c
+    class Processor < Relaton::Core::Processor
+      attr_reader :idtype
+
+      def initialize # rubocop:disable Lint/MissingSuper
+        @short = :relaton_w3c
+        @prefix = "W3C"
+        @defaultprefix = %r{^W3C\s}
+        @idtype = "W3C"
+        @datasets = %w[w3c-api]
+      end
+
+      def get(code, date, opts)
+        require_relative "../w3c"
+        Bibliography.get(code, date, opts)
+      end
+
+      def fetch_data(_source, opts)
+        require_relative "data_fetcher"
+        DataFetcher.fetch(**opts)
+      end
+
+      def from_xml(xml)
+        require_relative "../w3c"
+        Item.from_xml(xml)
+      end
+
+      def from_yaml(yaml)
+        require_relative "../w3c"
+        Item.from_yaml(yaml)
+      end
+
+      def grammar_hash
+        require_relative "../w3c"
+        @grammar_hash ||= Relaton::W3c.grammar_hash
+      end
+
+      def remove_index_file
+        require_relative "../w3c"
+        Relaton::Index.find_or_create(
+          :W3C, url: true, file: "#{INDEXFILE}.yaml"
+        ).remove_file
+      end
+    end
+  end
+end

data/lib/relaton/w3c/pubid.rb

@@ -0,0 +1,85 @@
+module Relaton
+  module W3c
+    # The legacy `index-v1` row shape.
+    #
+    # This is no longer on any lookup or crawl path in this gem: the flavor
+    # indexes and searches on `Pubid::W3c::Identifier` (see
+    # `Relaton::W3c::Docidentifier` and `INDEXFILE`). It is retained public
+    # only so `relaton-data-w3c`'s crawler can keep emitting the bespoke
+    # `index-v1` that released relaton v2 consumers read — the same reason
+    # `Relaton::Bipm::Id` was retained.
+    #
+    # Do not add call sites. Its decomposition differs from pubid's: it splits
+    # `year` from `date` and keeps a `/`-suffix in `suff`, where pubid folds
+    # both into the slug.
+    class PubId
+      PARTS = %i[code stage type year date suff].freeze
+
+      PARTS.each { |part| attr_accessor part }
+
+      def initialize(**parts)
+        PARTS.each { |part| send "#{part}=", parts[part] }
+      end
+
+      #
+      # Parse document identifier.
+      #
+      # @param [String] docnumber document identifier
+      #
+      # @return [Relaton::W3c::PubId] document identifier parts
+      #
+      def self.parse(docnumber) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        %r{
+          (?:^|/)(?:(?:(?<stage>WD|CRD|CR|PR|PER|REC|SPSD|OBSL|RET)|(?<type>D?NOTE|TR))[\s/-])?
+          (?<code>\w+(?:[+-][\w.]+)*?)
+          (?:-(?<year>(?:18|19|20)\d{2}))?
+          (?:-(?<date>\d{8}|\d{6}|\d{4}))?
+          (?:/(?<suff>\w+))?(?:$|/)
+        }xi =~ docnumber
+        entry = { code: code }
+        entry[:stage] = stage if stage
+        entry[:type] = type if type && type != "TR"
+        entry[:year] = year if year
+        entry[:date] = date if date
+        entry[:suff] = suff if suff
+        new(**entry)
+      end
+
+      #
+      # Compare document identifiers against Hash ID representation.
+      #
+      # @param [Hash] other hash of document identifier parts
+      #
+      # @return [Boolean] true if document identifiers are same
+      #
+      def ==(other) # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+        other[:code].casecmp?(code) && other[:stage] == stage && other[:type] == type &&
+          (year.nil? || other[:year] == year) && (date.nil? || other[:date] == date) &&
+          (suff.nil? || other[:suff]&.casecmp?(suff))
+      end
+
+      #
+      # Convert docidentifier identifier to hash.
+      #
+      # @return [Hash] hash of docidentifier parts
+      #
+      def to_hash
+        PARTS.each_with_object({}) { |part, hash| hash[part] = send part if send part }
+      end
+
+      #
+      # Render docidentifier as a string, the inverse of {.parse}.
+      #
+      # @return [String] document identifier
+      #
+      def to_s
+        prefix = stage || (type if type && type != "TR")
+        base = [prefix, code].compact.join("-")
+        base = "#{base}-#{year}" if year
+        base = "#{base}-#{date}" if date
+        base = "#{base}/#{suff}" if suff
+        base
+      end
+    end
+  end
+end